def test_invalid(self):
self.assertEqual(
ciso8601.parse_datetime_unaware('asdf'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('Z'),
None,
)
self.assertEqual(ciso8601.parse_datetime('2014-12-05asdfasdf'),
datetime.datetime(2014, 12, 5))
def test_invalid(self):
self.assertEqual(
ciso8601.parse_datetime_unaware('asdf'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('Z'),
None,
)
self.assertEqual(
ciso8601.parse_datetime('2014-12-05asdfasdf'),
datetime.datetime(2014, 12, 5)
)
def docker_link_sort_key(link):
"""Build a sort key to find the latest available Docker image.
To find one source collection for a Docker image referenced by
name or image id, the API server looks for a link with the most
recent `image_timestamp` property; then the most recent
`created_at` timestamp. This method generates a sort key for
Docker metadata links to sort them from least to most preferred.
"""
try:
image_timestamp = ciso8601.parse_datetime_unaware(link["properties"]["image_timestamp"])
except (KeyError, ValueError):
image_timestamp = EARLIEST_DATETIME
return (image_timestamp, ciso8601.parse_datetime_unaware(link["created_at"]))
def docker_link_sort_key(link):
"""Build a sort key to find the latest available Docker image.
To find one source collection for a Docker image referenced by
name or image id, the API server looks for a link with the most
recent `image_timestamp` property; then the most recent
`created_at` timestamp. This method generates a sort key for
Docker metadata links to sort them from least to most preferred.
"""
try:
image_timestamp = ciso8601.parse_datetime_unaware(
link['properties']['image_timestamp'])
except (KeyError, ValueError):
image_timestamp = EARLIEST_DATETIME
return (image_timestamp,
ciso8601.parse_datetime_unaware(link['created_at']))
def convertTime(t):
"""Parse Arvados timestamp to unix time."""
if not t:
return 0
try:
return calendar.timegm(ciso8601.parse_datetime_unaware(t).timetuple())
except (TypeError, ValueError):
return 0
def convertTime(t):
"""Parse Arvados timestamp to unix time."""
if not t:
return 0
try:
return calendar.timegm(ciso8601.parse_datetime_unaware(t).timetuple())
except (TypeError, ValueError):
return 0
def test_unaware(self):
expected = datetime.datetime(2014, 12, 5, 12, 30, 45, 123456)
self.assertEqual(ciso8601.parse_datetime('2014-12-05T12:30:45.123456'),
expected)
# parse_datetime_unaware ignores tz offset
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456Z'),
expected)
self.assertEqual(
ciso8601.parse_datetime_unaware(
'2014-12-05T12:30:45.123456+00:00'),
expected,
)
self.assertEqual(
ciso8601.parse_datetime_unaware(
'2014-12-05T12:30:45.123456-05:00'),
expected,
)
def dt_from_iso(profile):
dt = datetime.datetime.min
try:
last_seen = profile["$properties"]["$last_seen"]
try:
dt = ciso8601.parse_datetime_unaware(last_seen)
except NameError:
dt = datetime.datetime.strptime(last_seen, "%Y-%m-%dT%H:%M:%S")
except KeyError:
return dt
return dt
def test_unaware(self):
expected = datetime.datetime(2014, 12, 5, 12, 30, 45, 123456)
self.assertEqual(
ciso8601.parse_datetime('2014-12-05T12:30:45.123456'),
expected
)
# parse_datetime_unaware ignores tz offset
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456Z'),
expected
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456+00:00'),
expected,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456-05:00'),
expected,
)
def done(self, record):
outputs = {}
try:
container = self.arvrunner.api.containers().get(
uuid=record["container_uuid"]
).execute(num_retries=self.arvrunner.num_retries)
if container["state"] == "Complete":
rcode = container["exit_code"]
if self.successCodes and rcode in self.successCodes:
processStatus = "success"
elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
processStatus = "temporaryFail"
elif self.permanentFailCodes and rcode in self.permanentFailCodes:
processStatus = "permanentFail"
elif rcode == 0:
processStatus = "success"
else:
processStatus = "permanentFail"
else:
processStatus = "permanentFail"
if processStatus == "permanentFail":
logc = arvados.collection.CollectionReader(container["log"],
api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
label = self.arvrunner.label(self)
done.logtail(
logc, logger.error,
"%s (%s) error log:" % (label, record["uuid"]), maxlen=40)
if record["output_uuid"]:
if self.arvrunner.trash_intermediate or self.arvrunner.intermediate_output_ttl:
# Compute the trash time to avoid requesting the collection record.
trash_at = ciso8601.parse_datetime_unaware(record["modified_at"]) + datetime.timedelta(0, self.arvrunner.intermediate_output_ttl)
aftertime = " at %s" % trash_at.strftime("%Y-%m-%d %H:%M:%S UTC") if self.arvrunner.intermediate_output_ttl else ""
orpart = ", or" if self.arvrunner.trash_intermediate and self.arvrunner.intermediate_output_ttl else ""
oncomplete = " upon successful completion of the workflow" if self.arvrunner.trash_intermediate else ""
logger.info("%s Intermediate output %s (%s) will be trashed%s%s%s." % (
self.arvrunner.label(self), record["output_uuid"], container["output"], aftertime, orpart, oncomplete))
self.arvrunner.add_intermediate_output(record["output_uuid"])
if container["output"]:
outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
except WorkflowException as e:
logger.error("%s unable to collect output from %s:\n%s",
self.arvrunner.label(self), container["output"], e, exc_info=(e if self.arvrunner.debug else False))
processStatus = "permanentFail"
except Exception as e:
logger.exception("%s while getting output object: %s", self.arvrunner.label(self), e)
processStatus = "permanentFail"
finally:
self.output_callback(outputs, processStatus)
def done(self, record):
outputs = {}
try:
container = self.arvrunner.api.containers().get(
uuid=record["container_uuid"]
).execute(num_retries=self.arvrunner.num_retries)
if container["state"] == "Complete":
rcode = container["exit_code"]
if self.successCodes and rcode in self.successCodes:
processStatus = "success"
elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
processStatus = "temporaryFail"
elif self.permanentFailCodes and rcode in self.permanentFailCodes:
processStatus = "permanentFail"
elif rcode == 0:
processStatus = "success"
else:
processStatus = "permanentFail"
else:
processStatus = "permanentFail"
if processStatus == "permanentFail":
logc = arvados.collection.CollectionReader(container["log"],
api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
label = self.arvrunner.label(self)
done.logtail(
logc, logger.error,
"%s (%s) error log:" % (label, record["uuid"]), maxlen=40)
if record["output_uuid"]:
if self.arvrunner.trash_intermediate or self.arvrunner.intermediate_output_ttl:
# Compute the trash time to avoid requesting the collection record.
trash_at = ciso8601.parse_datetime_unaware(record["modified_at"]) + datetime.timedelta(0, self.arvrunner.intermediate_output_ttl)
aftertime = " at %s" % trash_at.strftime("%Y-%m-%d %H:%M:%S UTC") if self.arvrunner.intermediate_output_ttl else ""
orpart = ", or" if self.arvrunner.trash_intermediate and self.arvrunner.intermediate_output_ttl else ""
oncomplete = " upon successful completion of the workflow" if self.arvrunner.trash_intermediate else ""
logger.info("%s Intermediate output %s (%s) will be trashed%s%s%s." % (
self.arvrunner.label(self), record["output_uuid"], container["output"], aftertime, orpart, oncomplete))
self.arvrunner.add_intermediate_output(record["output_uuid"])
if container["output"]:
outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
except WorkflowException as e:
logger.error("%s unable to collect output from %s:\n%s",
self.arvrunner.label(self), container["output"], e, exc_info=(e if self.arvrunner.debug else False))
processStatus = "permanentFail"
except Exception as e:
logger.exception("%s while getting output object: %s", self.arvrunner.label(self), e)
processStatus = "permanentFail"
finally:
self.output_callback(outputs, processStatus)
def strip_datetime(value):
"""
Converts value to datetime if string or int.
"""
if isinstance(value, basestring):
try:
return ciso8601.parse_datetime_unaware(value)
except ValueError:
return
elif isinstance(value, integer_types):
try:
return datetime.datetime.utcfromtimestamp(value / 1e3)
except (ValueError, OverflowError, OSError):
return
def test_formats(self):
expected = datetime.datetime(2014, 2, 3)
self.assertEqual(ciso8601.parse_datetime('20140203'), expected)
self.assertEqual(ciso8601.parse_datetime('2014-02-03'), expected)
self.assertEqual(ciso8601.parse_datetime('2014-02'),
datetime.datetime(2014, 2, 1))
self.assertEqual(ciso8601.parse_datetime('2014-02T05'),
datetime.datetime(2014, 2, 1, 5))
self.assertEqual(ciso8601.parse_datetime('20140203T1035'),
datetime.datetime(2014, 2, 3, 10, 35))
self.assertEqual(ciso8601.parse_datetime('20140203T10:35'),
datetime.datetime(2014, 2, 3, 10, 35))
self.assertEqual(ciso8601.parse_datetime('20140203T103527'),
datetime.datetime(2014, 2, 3, 10, 35, 27))
self.assertEqual(ciso8601.parse_datetime('20140203T10:35:27'),
datetime.datetime(2014, 2, 3, 10, 35, 27))
self.assertEqual(ciso8601.parse_datetime('20140203T10:35:27.234'),
datetime.datetime(2014, 2, 3, 10, 35, 27, 234000))
self.assertEqual(ciso8601.parse_datetime('20140203T103527,234567'),
datetime.datetime(2014, 2, 3, 10, 35, 27, 234567))
self.assertEqual(
ciso8601.parse_datetime('20140203T103527.234567891234'),
datetime.datetime(2014, 2, 3, 10, 35, 27, 234567))
for leap_year in (1600, 2000, 2016):
self.assertEqual(
ciso8601.parse_datetime_unaware('{}-02-29'.format(leap_year)),
datetime.datetime(leap_year, 2, 29, 0, 0, 0, 0))
def test_invalid(self):
self.assertEqual(
ciso8601.parse_datetime_unaware('asdf'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-99-03'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-13-03'),
None,
)
self.assertEqual(
ciso8601.parse_datetime('20140203T24:35:27'),
None,
)
self.assertEqual(
ciso8601.parse_datetime('20140203T23:60:27'),
None,
)
self.assertEqual(
ciso8601.parse_datetime('20140203T23:35:61'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-01-32'),
None,
)
for non_leap_year in (1700, 1800, 1900, 2014):
self.assertEqual(
ciso8601.parse_datetime_unaware(
'{}-02-29'.format(non_leap_year)),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('Z'),
None,
)
self.assertEqual(ciso8601.parse_datetime('2014-12-05asdfasdf'),
datetime.datetime(2014, 12, 5))
import calendar
import ciso8601
import sys
fileinput = sys.argv[1]
fileoutput = sys.argv[2]
f = open(fileinput, 'rb')
fo = open(fileoutput, 'wb')
# go through each line of the file
for line in f:
bits = line.split(',')
# change second column
bits[0] = str(calendar.timegm(ciso8601.parse_datetime_unaware(bits[0]).timetuple()))
if "M" in bits[4]:
bits[4] = str(int(float(bits[4].split()[0])) * 1000000)
# join it back together and write it out
fo.write(','.join(bits))
f.close()
fo.close()
def parse_datetime(datetime_string):
return ciso8601.parse_datetime_unaware(datetime_string)
def test_invalid(self):
self.assertEqual(
ciso8601.parse_datetime_unaware('asdf'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-99-03'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-13-03'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-00-03'),
None,
)
self.assertEqual(
ciso8601.parse_datetime('20140203T24:35:27'),
None,
)
self.assertEqual(
ciso8601.parse_datetime('20140203T23:60:27'),
None,
)
self.assertEqual(
ciso8601.parse_datetime('20140203T23:35:61'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-01-32'),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('2014-06-31'),
None,
)
for non_leap_year in (1700, 1800, 1900, 2014):
self.assertEqual(
ciso8601.parse_datetime_unaware(
'{}-02-29'.format(non_leap_year)),
None,
)
self.assertEqual(
ciso8601.parse_datetime_unaware('Z'),
None,
)
self.assertEqual(ciso8601.parse_datetime('2014-12-05asdfasdf'),
datetime.datetime(2014, 12, 5))
parse = ciso8601.parse_datetime
none_or_dt = lambda *x: None if x[0] is None else datetime.datetime(*x)
check = lambda s, *x: self.assertEqual(parse(s), none_or_dt(*x))
check('20140203 04:05:06.123', 2014, 2, 3, 4, 5, 6, 123000)
check('20140203 04:05:06,123', 2014, 2, 3, 4, 5, 6, 123000)
check('20140203 04:05:0.123', None)
check('20140203 04:05:.123', None)
check('20140203 04:05:,123', None)
check('20140203 04:05:06.', 2014, 2, 3, 4, 5, 6)
check('20140203 0405:06.', 2014, 2, 3, 4, 5, 6)
check('20140203 040506.', 2014, 2, 3, 4, 5, 6)
check('20140203 04050.', None)
check('20140203 0405:0', None)
check('20140203 04:05:.', None)
check('20140203 04:05:,', None)
check('20140203 04::', None)
check('20140203 04:00:', 2014, 2, 3, 4)
check('20140203 04::01', None)
check('20140203 04:', 2014, 2, 3, 4)
check('2014-02-03', 2014, 2, 3)
check('2014-0-03', None)
check('2014--03', None)
check('2014-02', 2014, 2, 1)
check('2014--0', None)
check('2014--', None)
check('2014-', None)
check('2014', None)
check('20140203 040506.123', 2014, 2, 3, 4, 5, 6, 123000)
check('20140203 040506123', 2014, 2, 3, 4, 5, 6) # NB: drops usec
check('20140203 04050612', 2014, 2, 3, 4, 5, 6)
check('20140203 0405061', 2014, 2, 3, 4, 5, 6)
check('20140203 040506', 2014, 2, 3, 4, 5, 6)
check('20140203 04050', None)
check('20140203 0405', 2014, 2, 3, 4, 5)
check('20140203 040', None)
check('20140203 04', 2014, 2, 3, 4)
import calendar
import ciso8601
import sys
fileinput = sys.argv[1]
fileoutput = sys.argv[2]
f = open(fileinput, 'rb')
fo = open(fileoutput, 'wb')
# go through each line of the file
for line in f:
bits = line.split(',')
# change second column
bits[0] = str(
calendar.timegm(ciso8601.parse_datetime_unaware(bits[0]).timetuple()))
if "M" in bits[4]:
bits[4] = str(int(float(bits[4].split()[0])) * 1000000)
# join it back together and write it out
fo.write(','.join(bits))
f.close()
fo.close()
