def test_year_month(self):
pdt = PartialDateTime(2016, 8)
actual = pdt2num(pdt, 'days since 2016-08-20', 'gregorian')
expected = -19.
self.assertEqual(actual, expected)
def test_date_time(self):
pdt = PartialDateTime(2016, 8, 22, 14, 42, 11)
actual = pdt2num(pdt, 'days since 2016-08-20', 'gregorian')
expected = 2.6126273148148148
assert_almost_equal(actual, expected)
def test_year_month_end_period_360_day(self):
pdt = PartialDateTime(2016, 8)
actual = pdt2num(pdt,
'days since 2016-08-20',
'360_day',
start_of_period=False)
expected = 10.
self.assertEqual(actual, expected)
def create_database_file_object(metadata,
data_submission,
file_online=True,
file_version=None):
"""
Create a database entry for a data file
:param dict metadata: This file's metadata.
:param pdata_app.models.DataSubmission data_submission: The parent data
submission.
:param bool file_online: True if the file is online.
:param str file_version: The version string to apply to each file. The
string from the incoming directory name or the current date is used
if a string isn't supplied.
:returns:
"""
# get a fresh DB connection after exiting from parallel operation
django.db.connections.close_all()
time_units = Settings.get_solo().standard_time_units
if file_version:
version_string = file_version
else:
# find the version number from the date in the submission directory path
date_string = re.search(r'(?<=/incoming/)(\d{8})',
metadata['directory'])
if date_string:
date_string = date_string.group(0)
version_string = 'v' + date_string
else:
today = datetime.datetime.utcnow()
version_string = today.strftime('v%Y%m%d')
# if the file isn't online (e.g. loaded from JSON) then directory is blank
directory = metadata['directory'] if file_online else None
# create a data file. If the file already exists in the database with
# identical metadata then nothing happens. If the file exists but with
# slightly different metadata then django.db.utils.IntegrityError is
# raised
try:
data_file = DataFile.objects.create(
name=metadata['basename'],
incoming_name=metadata['basename'],
incoming_directory=metadata['directory'],
directory=directory,
size=metadata['filesize'],
project=metadata['project'],
institute=metadata['institute'],
climate_model=metadata['climate_model'],
activity_id=metadata['activity_id'],
experiment=metadata['experiment'],
variable_request=metadata['variable'],
data_request=metadata['data_request'],
frequency=metadata['frequency'],
rip_code=metadata['rip_code'],
start_time=pdt2num(metadata['start_date'], time_units,
metadata['calendar'])
if metadata['start_date'] else None,
end_time=pdt2num(metadata['end_date'],
time_units,
metadata['calendar'],
start_of_period=False)
if metadata['start_date'] else None,
time_units=time_units,
calendar=metadata['calendar'],
version=version_string,
data_submission=data_submission,
online=file_online,
grid=metadata.get('grid'),
tape_url=metadata.get('tape_url'))
except django.db.utils.IntegrityError as exc:
msg = ('Unable to submit file {}: {}'.format(metadata['basename'],
exc.__str__()))
logger.error(msg)
raise SubmissionError(msg)
if metadata['checksum_value']:
checksum = get_or_create(Checksum,
data_file=data_file,
checksum_value=metadata['checksum_value'],
checksum_type=metadata['checksum_type'])