def package_submission(basepath, recid, hep_submission_obj):
"""
Zips up a submission directory. This is in advance of its download
for example by users
:param basepath: path of directory containing all submission files
:param recid: the publication record ID
:param hep_submission_obj: the HEPSubmission object representing
the overall position
"""
if not os.path.exists(
os.path.join(current_app.config['CFG_DATADIR'], str(recid))):
os.makedirs(os.path.join(current_app.config['CFG_DATADIR'],
str(recid)))
version = hep_submission_obj.version
if version == 0:
version = 1
zip_location = os.path.join(
current_app.config['CFG_DATADIR'], str(recid),
current_app.config['SUBMISSION_FILE_NAME_PATTERN'].format(
recid, version))
if os.path.exists(zip_location):
os.remove(zip_location)
zipf = zipfile.ZipFile(zip_location, 'w')
os.chdir(basepath)
zipdir(".", zipf)
zipf.close()
def package_submission(basepath, recid, hep_submission_obj):
"""
Zips up a submission directory. This is in advance of its download
for example by users
:param basepath: path of directory containing all submission files
:param recid: the publication record ID
:param hep_submission_obj: the HEPSubmission object representing
the overall position
"""
if not os.path.exists(os.path.join(current_app.config['CFG_DATADIR'], str(recid))):
os.makedirs(os.path.join(current_app.config['CFG_DATADIR'], str(recid)))
version = hep_submission_obj.version
if version == 0:
version = 1
zip_location = os.path.join(
current_app.config['CFG_DATADIR'], str(recid),
current_app.config['SUBMISSION_FILE_NAME_PATTERN']
.format(recid, version))
if os.path.exists(zip_location):
os.remove(zip_location)
zipf = zipfile.ZipFile(zip_location, 'w')
os.chdir(basepath)
zipdir(".", zipf)
zipf.close()
def split_files(file_location, output_location,
archive_location=None):
"""
:param file_location: input yaml file location
:param output_location: output directory path
:param archive_location: if present will create a zipped
representation of the split files
"""
try:
try:
file_documents = yaml.load_all(open(file_location, 'r'), Loader=yaml.CSafeLoader)
except:
file_documents = yaml.safe_load_all(open(file_location, 'r'))
# make a submission directory where all the files will be stored.
# delete a directory in the event that it exists.
if os.path.exists(output_location):
shutil.rmtree(output_location)
os.makedirs(output_location)
with open(os.path.join(output_location, "submission.yaml"),
'w') as submission_yaml:
for document in file_documents:
if "record_ids" in document:
write_submission_yaml_block(
document, submission_yaml)
else:
file_name = document["name"].replace(' ', '') + ".yaml"
document["data_file"] = file_name
with open(os.path.join(output_location, file_name),
'w') as data_file:
yaml.add_representer(str, str_presenter)
yaml.dump(
{"independent_variables":
cleanup_data_yaml(
document["independent_variables"]),
"dependent_variables":
cleanup_data_yaml(
document["dependent_variables"])},
data_file, allow_unicode=True)
write_submission_yaml_block(document,
submission_yaml,
type="record")
if archive_location:
if os.path.exists(archive_location):
os.remove(archive_location)
zipf = zipfile.ZipFile(archive_location, 'w')
os.chdir(output_location)
zipdir(".", zipf)
zipf.close()
except Exception as e:
current_app.logger.exception(e)
current_app.logger.error(
'Error parsing {0}, {1}'.format(file_location, e.message))
def split_files(file_location, output_location, archive_location=None):
"""
:param file_location: input yaml file location
:param output_location: output directory path
:param archive_location: if present will create a zipped
representation of the split files
"""
last_updated = datetime.now()
try:
file_documents = yaml.load_all(open(file_location, 'r'), Loader=Loader)
# make a submission directory where all the files will be stored.
# delete a directory in the event that it exists.
if os.path.exists(output_location):
shutil.rmtree(output_location)
os.makedirs(output_location)
with open(os.path.join(output_location, "submission.yaml"),
'w') as submission_yaml:
for document in file_documents:
if not document:
continue
elif "name" not in document:
if "dateupdated" in document:
try:
last_updated = parse(document['dateupdated'],
dayfirst=True)
except ValueError as ve:
last_updated = datetime.now()
write_submission_yaml_block(document, submission_yaml)
else:
file_name = document["name"].replace(' ', '') + ".yaml"
document["data_file"] = file_name
with open(os.path.join(output_location, file_name),
'w') as data_file:
Dumper.add_representer(str, str_presenter)
yaml.dump(
{
"independent_variables":
cleanup_data_yaml(
document["independent_variables"]),
"dependent_variables":
cleanup_data_yaml(
document["dependent_variables"])
},
data_file,
allow_unicode=True,
Dumper=Dumper)
write_submission_yaml_block(document,
submission_yaml,
type="record")
if archive_location:
if os.path.exists(archive_location):
os.remove(archive_location)
zipf = zipfile.ZipFile(archive_location, 'w')
os.chdir(output_location)
zipdir(".", zipf)
zipf.close()
except yaml.scanner.ScannerError as se:
return se, last_updated
except yaml.parser.ParserError as pe:
return pe, last_updated
except Exception as e:
log.error('Error parsing %s, %s', file_location, e.message)
return e, last_updated
return None, last_updated