예제 #1
0
 def test_invalid(self):
     self.assertEqual(
         ciso8601.parse_datetime_unaware('asdf'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime_unaware('Z'),
         None,
     )
     self.assertEqual(ciso8601.parse_datetime('2014-12-05asdfasdf'),
                      datetime.datetime(2014, 12, 5))
예제 #2
0
파일: tests.py 프로젝트: JNRowe/ciso8601
 def test_invalid(self):
     self.assertEqual(
         ciso8601.parse_datetime_unaware('asdf'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime_unaware('Z'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime('2014-12-05asdfasdf'),
         datetime.datetime(2014, 12, 5)
     )
예제 #3
0
def docker_link_sort_key(link):
    """Build a sort key to find the latest available Docker image.

    To find one source collection for a Docker image referenced by
    name or image id, the API server looks for a link with the most
    recent `image_timestamp` property; then the most recent
    `created_at` timestamp.  This method generates a sort key for
    Docker metadata links to sort them from least to most preferred.
    """
    try:
        image_timestamp = ciso8601.parse_datetime_unaware(link["properties"]["image_timestamp"])
    except (KeyError, ValueError):
        image_timestamp = EARLIEST_DATETIME
    return (image_timestamp, ciso8601.parse_datetime_unaware(link["created_at"]))
예제 #4
0
def docker_link_sort_key(link):
    """Build a sort key to find the latest available Docker image.

    To find one source collection for a Docker image referenced by
    name or image id, the API server looks for a link with the most
    recent `image_timestamp` property; then the most recent
    `created_at` timestamp.  This method generates a sort key for
    Docker metadata links to sort them from least to most preferred.
    """
    try:
        image_timestamp = ciso8601.parse_datetime_unaware(
            link['properties']['image_timestamp'])
    except (KeyError, ValueError):
        image_timestamp = EARLIEST_DATETIME
    return (image_timestamp,
            ciso8601.parse_datetime_unaware(link['created_at']))
예제 #5
0
파일: fresh.py 프로젝트: chapmanb/arvados
def convertTime(t):
    """Parse Arvados timestamp to unix time."""
    if not t:
        return 0
    try:
        return calendar.timegm(ciso8601.parse_datetime_unaware(t).timetuple())
    except (TypeError, ValueError):
        return 0
예제 #6
0
파일: fresh.py 프로젝트: oliveralka/arvados
def convertTime(t):
    """Parse Arvados timestamp to unix time."""
    if not t:
        return 0
    try:
        return calendar.timegm(ciso8601.parse_datetime_unaware(t).timetuple())
    except (TypeError, ValueError):
        return 0
예제 #7
0
    def test_unaware(self):
        expected = datetime.datetime(2014, 12, 5, 12, 30, 45, 123456)
        self.assertEqual(ciso8601.parse_datetime('2014-12-05T12:30:45.123456'),
                         expected)

        # parse_datetime_unaware ignores tz offset
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456Z'),
            expected)
        self.assertEqual(
            ciso8601.parse_datetime_unaware(
                '2014-12-05T12:30:45.123456+00:00'),
            expected,
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware(
                '2014-12-05T12:30:45.123456-05:00'),
            expected,
        )
예제 #8
0
 def dt_from_iso(profile):
     dt = datetime.datetime.min
     try:
         last_seen = profile["$properties"]["$last_seen"]
         try:
             dt = ciso8601.parse_datetime_unaware(last_seen)
         except NameError:
             dt = datetime.datetime.strptime(last_seen, "%Y-%m-%dT%H:%M:%S")
     except KeyError:
         return dt
     return dt
예제 #9
0
파일: tests.py 프로젝트: JNRowe/ciso8601
    def test_unaware(self):
        expected = datetime.datetime(2014, 12, 5, 12, 30, 45, 123456)
        self.assertEqual(
            ciso8601.parse_datetime('2014-12-05T12:30:45.123456'),
            expected
        )

        # parse_datetime_unaware ignores tz offset
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456Z'),
            expected
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456+00:00'),
            expected,
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-12-05T12:30:45.123456-05:00'),
            expected,
        )
예제 #10
0
    def done(self, record):
        outputs = {}
        try:
            container = self.arvrunner.api.containers().get(
                uuid=record["container_uuid"]
            ).execute(num_retries=self.arvrunner.num_retries)
            if container["state"] == "Complete":
                rcode = container["exit_code"]
                if self.successCodes and rcode in self.successCodes:
                    processStatus = "success"
                elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
                    processStatus = "temporaryFail"
                elif self.permanentFailCodes and rcode in self.permanentFailCodes:
                    processStatus = "permanentFail"
                elif rcode == 0:
                    processStatus = "success"
                else:
                    processStatus = "permanentFail"
            else:
                processStatus = "permanentFail"

            if processStatus == "permanentFail":
                logc = arvados.collection.CollectionReader(container["log"],
                                                           api_client=self.arvrunner.api,
                                                           keep_client=self.arvrunner.keep_client,
                                                           num_retries=self.arvrunner.num_retries)
                label = self.arvrunner.label(self)
                done.logtail(
                    logc, logger.error,
                    "%s (%s) error log:" % (label, record["uuid"]), maxlen=40)

            if record["output_uuid"]:
                if self.arvrunner.trash_intermediate or self.arvrunner.intermediate_output_ttl:
                    # Compute the trash time to avoid requesting the collection record.
                    trash_at = ciso8601.parse_datetime_unaware(record["modified_at"]) + datetime.timedelta(0, self.arvrunner.intermediate_output_ttl)
                    aftertime = " at %s" % trash_at.strftime("%Y-%m-%d %H:%M:%S UTC") if self.arvrunner.intermediate_output_ttl else ""
                    orpart = ", or" if self.arvrunner.trash_intermediate and self.arvrunner.intermediate_output_ttl else ""
                    oncomplete = " upon successful completion of the workflow" if self.arvrunner.trash_intermediate else ""
                    logger.info("%s Intermediate output %s (%s) will be trashed%s%s%s." % (
                        self.arvrunner.label(self), record["output_uuid"], container["output"], aftertime, orpart, oncomplete))
                self.arvrunner.add_intermediate_output(record["output_uuid"])

            if container["output"]:
                outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
        except WorkflowException as e:
            logger.error("%s unable to collect output from %s:\n%s",
                         self.arvrunner.label(self), container["output"], e, exc_info=(e if self.arvrunner.debug else False))
            processStatus = "permanentFail"
        except Exception as e:
            logger.exception("%s while getting output object: %s", self.arvrunner.label(self), e)
            processStatus = "permanentFail"
        finally:
            self.output_callback(outputs, processStatus)
예제 #11
0
    def done(self, record):
        outputs = {}
        try:
            container = self.arvrunner.api.containers().get(
                uuid=record["container_uuid"]
            ).execute(num_retries=self.arvrunner.num_retries)
            if container["state"] == "Complete":
                rcode = container["exit_code"]
                if self.successCodes and rcode in self.successCodes:
                    processStatus = "success"
                elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
                    processStatus = "temporaryFail"
                elif self.permanentFailCodes and rcode in self.permanentFailCodes:
                    processStatus = "permanentFail"
                elif rcode == 0:
                    processStatus = "success"
                else:
                    processStatus = "permanentFail"
            else:
                processStatus = "permanentFail"

            if processStatus == "permanentFail":
                logc = arvados.collection.CollectionReader(container["log"],
                                                           api_client=self.arvrunner.api,
                                                           keep_client=self.arvrunner.keep_client,
                                                           num_retries=self.arvrunner.num_retries)
                label = self.arvrunner.label(self)
                done.logtail(
                    logc, logger.error,
                    "%s (%s) error log:" % (label, record["uuid"]), maxlen=40)

            if record["output_uuid"]:
                if self.arvrunner.trash_intermediate or self.arvrunner.intermediate_output_ttl:
                    # Compute the trash time to avoid requesting the collection record.
                    trash_at = ciso8601.parse_datetime_unaware(record["modified_at"]) + datetime.timedelta(0, self.arvrunner.intermediate_output_ttl)
                    aftertime = " at %s" % trash_at.strftime("%Y-%m-%d %H:%M:%S UTC") if self.arvrunner.intermediate_output_ttl else ""
                    orpart = ", or" if self.arvrunner.trash_intermediate and self.arvrunner.intermediate_output_ttl else ""
                    oncomplete = " upon successful completion of the workflow" if self.arvrunner.trash_intermediate else ""
                    logger.info("%s Intermediate output %s (%s) will be trashed%s%s%s." % (
                        self.arvrunner.label(self), record["output_uuid"], container["output"], aftertime, orpart, oncomplete))
                self.arvrunner.add_intermediate_output(record["output_uuid"])

            if container["output"]:
                outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
        except WorkflowException as e:
            logger.error("%s unable to collect output from %s:\n%s",
                         self.arvrunner.label(self), container["output"], e, exc_info=(e if self.arvrunner.debug else False))
            processStatus = "permanentFail"
        except Exception as e:
            logger.exception("%s while getting output object: %s", self.arvrunner.label(self), e)
            processStatus = "permanentFail"
        finally:
            self.output_callback(outputs, processStatus)
예제 #12
0
 def strip_datetime(value):
     """
     Converts value to datetime if string or int.
     """
     if isinstance(value, basestring):
         try:
             return ciso8601.parse_datetime_unaware(value)
         except ValueError:
             return
     elif isinstance(value, integer_types):
         try:
             return datetime.datetime.utcfromtimestamp(value / 1e3)
         except (ValueError, OverflowError, OSError):
             return
예제 #13
0
    def test_formats(self):
        expected = datetime.datetime(2014, 2, 3)
        self.assertEqual(ciso8601.parse_datetime('20140203'), expected)
        self.assertEqual(ciso8601.parse_datetime('2014-02-03'), expected)

        self.assertEqual(ciso8601.parse_datetime('2014-02'),
                         datetime.datetime(2014, 2, 1))

        self.assertEqual(ciso8601.parse_datetime('2014-02T05'),
                         datetime.datetime(2014, 2, 1, 5))

        self.assertEqual(ciso8601.parse_datetime('20140203T1035'),
                         datetime.datetime(2014, 2, 3, 10, 35))

        self.assertEqual(ciso8601.parse_datetime('20140203T10:35'),
                         datetime.datetime(2014, 2, 3, 10, 35))

        self.assertEqual(ciso8601.parse_datetime('20140203T103527'),
                         datetime.datetime(2014, 2, 3, 10, 35, 27))

        self.assertEqual(ciso8601.parse_datetime('20140203T10:35:27'),
                         datetime.datetime(2014, 2, 3, 10, 35, 27))

        self.assertEqual(ciso8601.parse_datetime('20140203T10:35:27.234'),
                         datetime.datetime(2014, 2, 3, 10, 35, 27, 234000))

        self.assertEqual(ciso8601.parse_datetime('20140203T103527,234567'),
                         datetime.datetime(2014, 2, 3, 10, 35, 27, 234567))

        self.assertEqual(
            ciso8601.parse_datetime('20140203T103527.234567891234'),
            datetime.datetime(2014, 2, 3, 10, 35, 27, 234567))
        for leap_year in (1600, 2000, 2016):
            self.assertEqual(
                ciso8601.parse_datetime_unaware('{}-02-29'.format(leap_year)),
                datetime.datetime(leap_year, 2, 29, 0, 0, 0, 0))
예제 #14
0
 def test_invalid(self):
     self.assertEqual(
         ciso8601.parse_datetime_unaware('asdf'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime_unaware('2014-99-03'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime_unaware('2014-13-03'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime('20140203T24:35:27'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime('20140203T23:60:27'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime('20140203T23:35:61'),
         None,
     )
     self.assertEqual(
         ciso8601.parse_datetime_unaware('2014-01-32'),
         None,
     )
     for non_leap_year in (1700, 1800, 1900, 2014):
         self.assertEqual(
             ciso8601.parse_datetime_unaware(
                 '{}-02-29'.format(non_leap_year)),
             None,
         )
     self.assertEqual(
         ciso8601.parse_datetime_unaware('Z'),
         None,
     )
     self.assertEqual(ciso8601.parse_datetime('2014-12-05asdfasdf'),
                      datetime.datetime(2014, 12, 5))
예제 #15
0
import calendar
import ciso8601
import sys

fileinput = sys.argv[1]
fileoutput = sys.argv[2]

f = open(fileinput, 'rb')
fo = open(fileoutput, 'wb')

# go through each line of the file
for line in f:
    bits = line.split(',')
    # change second column
    bits[0] = str(calendar.timegm(ciso8601.parse_datetime_unaware(bits[0]).timetuple()))
    if "M" in bits[4]:
        bits[4] = str(int(float(bits[4].split()[0])) * 1000000)

    # join it back together and write it out
    fo.write(','.join(bits))

f.close()
fo.close()
예제 #16
0
파일: compat.py 프로젝트: w1r2p1/betfair
 def parse_datetime(datetime_string):
     return ciso8601.parse_datetime_unaware(datetime_string)
예제 #17
0
    def test_invalid(self):
        self.assertEqual(
            ciso8601.parse_datetime_unaware('asdf'),
            None,
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-99-03'),
            None,
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-13-03'),
            None,
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-00-03'),
            None,
        )
        self.assertEqual(
            ciso8601.parse_datetime('20140203T24:35:27'),
            None,
        )
        self.assertEqual(
            ciso8601.parse_datetime('20140203T23:60:27'),
            None,
        )
        self.assertEqual(
            ciso8601.parse_datetime('20140203T23:35:61'),
            None,
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-01-32'),
            None,
        )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('2014-06-31'),
            None,
        )
        for non_leap_year in (1700, 1800, 1900, 2014):
            self.assertEqual(
                ciso8601.parse_datetime_unaware(
                    '{}-02-29'.format(non_leap_year)),
                None,
            )
        self.assertEqual(
            ciso8601.parse_datetime_unaware('Z'),
            None,
        )
        self.assertEqual(ciso8601.parse_datetime('2014-12-05asdfasdf'),
                         datetime.datetime(2014, 12, 5))

        parse = ciso8601.parse_datetime
        none_or_dt = lambda *x: None if x[0] is None else datetime.datetime(*x)
        check = lambda s, *x: self.assertEqual(parse(s), none_or_dt(*x))

        check('20140203 04:05:06.123', 2014, 2, 3, 4, 5, 6, 123000)
        check('20140203 04:05:06,123', 2014, 2, 3, 4, 5, 6, 123000)
        check('20140203 04:05:0.123', None)
        check('20140203 04:05:.123', None)
        check('20140203 04:05:,123', None)
        check('20140203 04:05:06.', 2014, 2, 3, 4, 5, 6)
        check('20140203 0405:06.', 2014, 2, 3, 4, 5, 6)
        check('20140203 040506.', 2014, 2, 3, 4, 5, 6)
        check('20140203 04050.', None)
        check('20140203 0405:0', None)
        check('20140203 04:05:.', None)
        check('20140203 04:05:,', None)
        check('20140203 04::', None)
        check('20140203 04:00:', 2014, 2, 3, 4)
        check('20140203 04::01', None)
        check('20140203 04:', 2014, 2, 3, 4)

        check('2014-02-03', 2014, 2, 3)
        check('2014-0-03', None)
        check('2014--03', None)
        check('2014-02', 2014, 2, 1)
        check('2014--0', None)
        check('2014--', None)
        check('2014-', None)
        check('2014', None)

        check('20140203 040506.123', 2014, 2, 3, 4, 5, 6, 123000)
        check('20140203 040506123', 2014, 2, 3, 4, 5, 6)  # NB: drops usec
        check('20140203 04050612', 2014, 2, 3, 4, 5, 6)
        check('20140203 0405061', 2014, 2, 3, 4, 5, 6)
        check('20140203 040506', 2014, 2, 3, 4, 5, 6)
        check('20140203 04050', None)
        check('20140203 0405', 2014, 2, 3, 4, 5)
        check('20140203 040', None)
        check('20140203 04', 2014, 2, 3, 4)
예제 #18
0
import calendar
import ciso8601
import sys

fileinput = sys.argv[1]
fileoutput = sys.argv[2]

f = open(fileinput, 'rb')
fo = open(fileoutput, 'wb')

# go through each line of the file
for line in f:
    bits = line.split(',')
    # change second column
    bits[0] = str(
        calendar.timegm(ciso8601.parse_datetime_unaware(bits[0]).timetuple()))
    if "M" in bits[4]:
        bits[4] = str(int(float(bits[4].split()[0])) * 1000000)

    # join it back together and write it out
    fo.write(','.join(bits))

f.close()
fo.close()