def test_parse_pi_xml_02(self):
"""Parse a file having comment elements."""
source = os.path.join(DATA_DIR, "GDresults_dam.xml")
reader = PiXmlReader(source)
for md, df in reader.bulk_get_series(chunk_size=5):
pass
self.assertTrue(True)
def import_pi_xml(src, usr_id):
logger.info("[x] Importing %r" % src)
reader = PiXmlReader(src)
usr = User.objects.get(pk=usr_id)
for md, df in reader.get_series():
loc = md['header']['locationId'] # required
para = md['header']['parameterId'] # required
unit = md['header']['timeStep']['@unit'] # required
div = md['header']['timeStep'].get('@divider', '') # optional
mul = md['header']['timeStep'].get('@multiplier', '') # optional
remote_id = loc + '::' + para + '::' + unit + '::' + div + '::' + mul
ts = get_auth(usr, remote_id)
if ts is False:
data_move(src, ERROR_file)
logger.error(
'[x] File:--%r-- has been rejected because of authorization' %
src)
raise Exception("[x] %r _FAILED to be imported" % (src))
else:
try:
del df['flag']
ts.set_events(df)
ts.save()
logger.debug("[x] %r _written" % (src))
except:
logger.error(
"[x] %r _FAILED to be written to cassandra" % (src))
data_move(src, ERROR_file)
raise Exception('piXML file: %r ERROR to convert!' % src)
data_move(src, OK_file)
logger.info('[x] File:--%r-- has been successfully imported' % src)
def test_parse_pi_xml_01(self):
"""Parse a file."""
source = os.path.join(DATA_DIR, "time_series.xml")
reader = PiXmlReader(source)
for md, df in reader.bulk_get_series(chunk_size=5):
pass
self.assertTrue(True)
def test_parse_pi_xml_03(self):
"""Parse a file with timeZone element."""
source = os.path.join(DATA_DIR, "time_series.xml")
reader = PiXmlReader(source)
tz = reader.get_tz()
self.assertEqual(1.0, tz)
def test_parse_pi_xml_09(self):
"""Parse a file without events ."""
source = os.path.join(DATA_DIR, "no_events.xml")
reader = PiXmlReader(source)
for md, df in reader.get_series():
self.assertEqual(None, df)
def test_parse_pi_xml_05(self):
"""Parse a file without timeZone element."""
source = os.path.join(DATA_DIR, "no_tz.xml")
reader = PiXmlReader(source)
tz = reader.get_tz()
self.assertEqual(None, tz)
def test_parse_pi_xml_04(self):
"""Parse a file with empty timeZone element."""
source = os.path.join(DATA_DIR, "empty_tz.xml")
reader = PiXmlReader(source)
tz = reader.get_tz()
self.assertEqual(0.0, tz)
def test_parse_pi_xml_01(self):
source = os.path.join(DATA_DIR, "time_series.xml")
reader = PiXmlReader(source)
for md, df in reader.get_series():
pass
self.assertTrue(True)
def handle(self, *args, **options):
# source is a pi.xml file that serves as a template: its headers are
# used, but any events are ignored. The `comment` element of each
# series is expected to contain a ddsc uuid.
try:
source = args[0]
except IndexError:
self.stderr.write(self.help)
return
# destination is the resulting pi.xml file. If no destination is
# given, output is written to stdout. Allow for strftime based
# formatting of the filename. See: http://docs.python.org/2/
# library/datetime.html#strftime-strptime-behavior
try:
utcnow = timezone('UTC').localize(datetime.utcnow())
tznow = utcnow.astimezone(timezone(options.get('timezone')))
filename = tznow.strftime(options.get('file'))
destination = open(filename, 'w')
except TypeError:
destination = self.stdout
# To limit the events to be returned, `start` and `end` can be used.
# They can either be expressed as an ISO 8601 UTC datetime string
# or as a timedelta relative to now, for example:
#
# --start=2012-01-01T00:00:00Z --end=2013-01-01T00:00:00Z
# --start="{'days': -7}" --end="{}"
start = options.get('start')
if start is not None:
try:
start = datetime.strptime(start, ISO8601)
except:
try:
start = utcnow + timedelta(**literal_eval(start))
except:
logger.error("Cannot parse option `start`")
start = None
end = options.get('end')
if end is not None:
try:
end = datetime.strptime(end, ISO8601)
except:
try:
end = utcnow + timedelta(**literal_eval(end))
except:
logger.error("Cannot parse option `end`")
end = None
reader = PiXmlReader(source)
writer = PiXmlWriter(reader.get_tz())
for md, dummy in reader.get_series():
try:
uuid = md.pop('comment')
timeseries = Timeseries.objects.get(uuid=uuid)
df = timeseries.get_events(start=start, end=end)
writer.set_series(md, df)
except Timeseries.DoesNotExist:
logger.warning("Timeseries {0} does not exist".format(uuid))
df = pd.DataFrame()
writer.set_series(md, df)
except Exception as e:
logger.error(e)
writer.write(destination)
