def test_parse_pi_xml_02(self): """Parse a file having comment elements.""" source = os.path.join(DATA_DIR, "GDresults_dam.xml") reader = PiXmlReader(source) for md, df in reader.bulk_get_series(chunk_size=5): pass self.assertTrue(True)
def import_pi_xml(src, usr_id): logger.info("[x] Importing %r" % src) reader = PiXmlReader(src) usr = User.objects.get(pk=usr_id) for md, df in reader.get_series(): loc = md['header']['locationId'] # required para = md['header']['parameterId'] # required unit = md['header']['timeStep']['@unit'] # required div = md['header']['timeStep'].get('@divider', '') # optional mul = md['header']['timeStep'].get('@multiplier', '') # optional remote_id = loc + '::' + para + '::' + unit + '::' + div + '::' + mul ts = get_auth(usr, remote_id) if ts is False: data_move(src, ERROR_file) logger.error( '[x] File:--%r-- has been rejected because of authorization' % src) raise Exception("[x] %r _FAILED to be imported" % (src)) else: try: del df['flag'] ts.set_events(df) ts.save() logger.debug("[x] %r _written" % (src)) except: logger.error( "[x] %r _FAILED to be written to cassandra" % (src)) data_move(src, ERROR_file) raise Exception('piXML file: %r ERROR to convert!' % src) data_move(src, OK_file) logger.info('[x] File:--%r-- has been successfully imported' % src)
def test_parse_pi_xml_01(self): """Parse a file.""" source = os.path.join(DATA_DIR, "time_series.xml") reader = PiXmlReader(source) for md, df in reader.bulk_get_series(chunk_size=5): pass self.assertTrue(True)
def test_parse_pi_xml_03(self): """Parse a file with timeZone element.""" source = os.path.join(DATA_DIR, "time_series.xml") reader = PiXmlReader(source) tz = reader.get_tz() self.assertEqual(1.0, tz)
def test_parse_pi_xml_09(self): """Parse a file without events .""" source = os.path.join(DATA_DIR, "no_events.xml") reader = PiXmlReader(source) for md, df in reader.get_series(): self.assertEqual(None, df)
def test_parse_pi_xml_05(self): """Parse a file without timeZone element.""" source = os.path.join(DATA_DIR, "no_tz.xml") reader = PiXmlReader(source) tz = reader.get_tz() self.assertEqual(None, tz)
def test_parse_pi_xml_04(self): """Parse a file with empty timeZone element.""" source = os.path.join(DATA_DIR, "empty_tz.xml") reader = PiXmlReader(source) tz = reader.get_tz() self.assertEqual(0.0, tz)
def test_parse_pi_xml_01(self): source = os.path.join(DATA_DIR, "time_series.xml") reader = PiXmlReader(source) for md, df in reader.get_series(): pass self.assertTrue(True)
def handle(self, *args, **options): # source is a pi.xml file that serves as a template: its headers are # used, but any events are ignored. The `comment` element of each # series is expected to contain a ddsc uuid. try: source = args[0] except IndexError: self.stderr.write(self.help) return # destination is the resulting pi.xml file. If no destination is # given, output is written to stdout. Allow for strftime based # formatting of the filename. See: http://docs.python.org/2/ # library/datetime.html#strftime-strptime-behavior try: utcnow = timezone('UTC').localize(datetime.utcnow()) tznow = utcnow.astimezone(timezone(options.get('timezone'))) filename = tznow.strftime(options.get('file')) destination = open(filename, 'w') except TypeError: destination = self.stdout # To limit the events to be returned, `start` and `end` can be used. # They can either be expressed as an ISO 8601 UTC datetime string # or as a timedelta relative to now, for example: # # --start=2012-01-01T00:00:00Z --end=2013-01-01T00:00:00Z # --start="{'days': -7}" --end="{}" start = options.get('start') if start is not None: try: start = datetime.strptime(start, ISO8601) except: try: start = utcnow + timedelta(**literal_eval(start)) except: logger.error("Cannot parse option `start`") start = None end = options.get('end') if end is not None: try: end = datetime.strptime(end, ISO8601) except: try: end = utcnow + timedelta(**literal_eval(end)) except: logger.error("Cannot parse option `end`") end = None reader = PiXmlReader(source) writer = PiXmlWriter(reader.get_tz()) for md, dummy in reader.get_series(): try: uuid = md.pop('comment') timeseries = Timeseries.objects.get(uuid=uuid) df = timeseries.get_events(start=start, end=end) writer.set_series(md, df) except Timeseries.DoesNotExist: logger.warning("Timeseries {0} does not exist".format(uuid)) df = pd.DataFrame() writer.set_series(md, df) except Exception as e: logger.error(e) writer.write(destination)