def parse(self, fileObj, name):
try:
lines = fileObj.readlines()
lineCounter = 0
for line in lines:
lineCounter = lineCounter + 1
s = line.strip(' \t\n\r')
row = s.split(";")
if s != '' and len(row) >= (self.max_index + 1):
date = self.parseDate(row)
if str(row[self.config['rowid'][1]]) == str(
self.config['rowid'][0]):
self.setEndPosition(date)
values = {}
value = row[self.config["column"]]
if value.find('[') >= 0:
value = '-999.9'
values[self.config["observedProperty"]] = value
self.addObservation(raw2csv.Observation(date, values))
except Exception as ex:
traceback.print_exc()
raise ex
def parse(self, fileObj, fileName):
# print "Filename: %s" % fileName
cnt = 0
for line in fileObj.readlines():
cnt = cnt + 1
# Skipping header rows if present in configuration
if "headrows" in self.config and cnt <= self.config['headrows']:
continue
elif "stopat" in self.config:
if isinstance(self.config["stopat"],
str) and (line.find(self.config["stopat"]) >= 0):
break
elif isinstance(self.config["stopat"], int):
pass
else:
pass
# Line splitting
columns = line.split(self.config['separator'])
try:
date = self.parseDate(columns)
values = {}
for obs in self.config["observations"]:
values[obs["observedProperty"]] = columns[obs["column"]]
self.addObservation(raw2csv.Observation(date, values))
self.setEndPosition(date)
except Exception as e:
print "%s [%s]:%s" % (fileName, cnt, line)
print traceback.print_exc()
raise e
self.setEndPositionFromFilename(fileName)
def parse(self, fileObj, fileName):
skipline = fileName.split("_")[0]
dateformat = "%Y-%m-%d %H:%M:%S"
# STS procedures have only one observed property
op = self.getDefinitions()[1]
cnt = 0
linetmp = ''
try:
for line in fileObj.readlines():
cnt = cnt + 1
linetmp = line
if line.find(skipline) > -1 or line.find('data') > -1 or len(
line) == 0:
continue
pair = line.split(";")
if len(pair) != 2:
continue
try:
data = datetime.strptime(pair[0], dateformat)
except Exception as e:
continue
val = {op: pair[1]}
data = datetime.strptime(pair[0], dateformat)
if "tz" in self.config:
data = self.getDateTimeWithTimeZone(
data, self.config["tz"])
# Removing seconds from date
if 'rmseconds' not in self.config or self.config[
'rmseconds'] is True:
data = datetime(data.year,
data.month,
data.day,
data.hour,
data.minute,
0,
tzinfo=data.tzinfo)
self.setEndPosition(data)
self.addObservation(raw2csv.Observation(data, val))
except Exception as e:
raise Exception("Error at row: %s:%s\n%s" % (cnt, linetmp, e))
def parse(self, fileObj, name):
lines = fileObj.readlines()
lineCounter = 0
for line in lines:
lineCounter = lineCounter + 1
s = line.strip(' \t\n\r')
arr = s.split(",")
if s != '' and len(arr) > 0:
date = self.getDate(arr[self.config['date'][0]],
arr[self.config['date'][1]],
arr[self.config['date'][2]])
self.setEndPosition(date)
if str(arr[0]) == str(self.config['rowid']):
if len(self.config['date']) == 3:
date = self.getDate(arr[self.config['date'][0]],
arr[self.config['date'][1]],
arr[self.config['date'][2]])
elif len(self.config['date']) == 4:
date = self.getDate(arr[self.config['date'][0]],
arr[self.config['date'][1]],
arr[self.config['date'][2]],
arr[self.config['date'][3]])
else:
raise raw2csv.IstSOSError(
"Date configuration mismatch.")
self.setEndPosition(date)
values = {}
if "column" in self.config:
# It happens that some lines are cutted so the line is skipped
try:
arr[self.config["column"]]
except:
print "Error in line %s: %s" % (lineCounter, arr)
continue
values[self.config["observedProperty"]] = arr[
self.config["column"]]
else:
values[self.config["observedProperty"]] = float(
self.config["value"])
self.addObservation(raw2csv.Observation(date, values))
def parse(self, fileObj, fileName):
isHead = False
isData = False
cnt = 0
for line in fileObj.readlines():
cnt = cnt+1
try:
# Special characters
# https://it.wikipedia.org/wiki/Carattere_di_controllo#Tavole
# NUL file nullo
if line.find('\x00') >= 0:
break
# SOH indica una riga di intestazione con la data
# di inizio dei dati
if line.find('\x01TI') >= 0:
line = line.replace('\x01', '')
isHead = True
else:
isHead = False
# STX indica l'inizio di un blocco con i dati
if line.find('\x02') >= 0:
line = line.replace('\x02', '')
isData = True
# ETX indica la fine del blocco con i dati
if line.find('\x03') >= 0:
line = line.replace('\x03', '')
isData = False
# EOT indica la fine del file
if line.find('\x04') >= 0:
line = line.replace('\x04', '')
break
line = line.split()
if isHead:
# Estrazione dell'anno e i minuti dall'intestazione
# \/ \/
# ['TI', '2012', '183721', '000000', 'HBTIa', '14', 'KERN',
# 'TL-1', 'SN:557']
year = datetime.strptime(line[1], '%Y').year
startMinutes = line[2]
if isData and line[0] in ['D', 'd', 'o']:
dataMinutes = line[1]
# Controllo del capodanno
if int(startMinutes) > int(dataMinutes):
year = year + 1
d = self.minutesdate(year, dataMinutes)
if self.getDSEndPosition() is not None and d < self.getDSEndPosition():
self.addWarning(
"File %s is inserting measurements "
" prior to the EndPosition, skipping line." % (
fileName
)
)
self.setEndPosition(d)
val = {}
if isinstance(self.config["observations"], list):
for obs in self.config["observations"]:
val[obs['observedProperty']] = line[obs['column']]
else:
val[
self.config["observations"]['observedProperty']
] = line[self.config["observations"]['column']]
self.addObservation(
raw2csv.Observation(d, val)
)
except Exception as e:
self.log("%s:%s\n Line: %s" % (fileName, cnt, line))
self.log(traceback.print_exc())
raise e
def parse(self, fileObj, fileName):
isHead = False
isData = False
cnt = 0
for line in fileObj.readlines():
cnt = cnt + 1
try:
# Special characters https://it.wikipedia.org/wiki/Carattere_di_controllo#Tavole
# SOH indica una riga di intestazione con la data di inizio dei dati
if line.find('\x01TI') >= 0:
line = line.replace('\x01', '')
isHead = True
else:
isHead = False
# STX indica l'inizio di un blocco con i dati
if line.find('\x02') >= 0:
line = line.replace('\x02', '')
isData = True
# ETX indica la fine del blocco con i dati
if line.find('\x03') >= 0:
line = line.replace('\x03', '')
isData = False
# EOT indica la fine del file
if line.find('\x04') >= 0:
line = line.replace('\x04', '')
break
line = line.split()
if isHead:
# Estrazione dell'anno e i minuti dalla riga di intestazione
# \/ \/
# ['TI', '2012', '183721', '000000', 'HBTIa', '14', 'KERN', 'TL-1', 'SN:557']
year = datetime.strptime(line[1], '%Y').year
startMinutes = line[2]
if isData and line[0] in ['D', 'd', 'o']:
dataMinutes = line[1]
# Controllo del capodanno
if int(startMinutes) > int(dataMinutes):
year = year + 1
d = self.minutesdate(year, dataMinutes)
#d = datetime(d.year, d.month, d.day, d.hour, d.minute, d.second,
# d.microsecond, tzinfo=timezone("CET"))
self.setEndPosition(d)
val = {}
if type(self.config["observations"]) == type([]):
for obs in self.config["observations"]:
val[obs['observedProperty']] = line[obs['column']]
else:
val[self.config["observations"]
['observedProperty']] = line[
self.config["observations"]['column']]
self.addObservation(raw2csv.Observation(d, val))
except Exception as e:
self.log("%s:%s\n Line: %s" % (fileName, cnt, line))
self.log(traceback.print_exc())
raise e