def loadAll(config):
rows = []
reader = None
for path in sorted(glob.iglob(config.data_folder + '/*.csv')):
with codecs.open(path, 'r', encoding='latin1') as in_file:
reader = csv.reader(in_file)
for row in reader:
if row:
if not data.not_data_point(row):
row.pop()
for i in range(1, 5):
if row[i] != 'n/a':
row[i] = float(row[i])
else:
row[i] = 0.0
rows.append(row)
if reader is None:
click.echo('No csv files found in ' + config.data_folder + ', nothing to do.')
sys.exit(4)
# Deduplication.
seen = set()
rows = [x for x in rows if not (x[0] in seen or seen.add(x[0]))]
return sorted(rows, key=time.datetime_from_row)
def loadAll(config):
rows = []
reader = None
for path in sorted(glob.iglob(config.data_folder + '/*.csv')):
with codecs.open(path, 'r', encoding='latin1') as in_file:
reader = csv.reader(in_file)
for row in reader:
if row:
if not data.not_data_point(row):
row.pop()
for i in range(1, 5):
if row[i] != 'n/a':
row[i] = float(row[i])
else:
row[i] = 0.0
rows.append(row)
if reader is None:
click.echo('No csv files found in ' + config.data_folder +
', nothing to do.')
sys.exit(4)
# Deduplication.
seen = set()
rows = [x for x in rows if not (x[0] in seen or seen.add(x[0]))]
return sorted(rows, key=time.datetime_from_row)
def datetime_from_row(row, cutoff=None):
"""Get the full date and time from the data point row."""
from comet.data import not_data_point
if not_data_point(row):
return datetime.datetime(1970, 1, 1)
return datetime_from_field(row[0], cutoff)
def datetime_from_row(row, cutoff=None):
"""Get the full date and time from the data point row."""
from comet.data import not_data_point
if not_data_point(row):
return datetime.datetime(1970, 1, 1)
return datetime_from_field(row[0], cutoff)
def fetch(config, url):
"""Fetches and stores metrics from Sensor at the URL given."""
new_path = os.path.join(
config.data_folder,
datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S.csv'))
new_temp_path = new_path + 'temp'
if not url.startswith('http://'):
url = 'http://' + url
url += '/export.csv'
if config.verbose:
click.echo('Fetching data from ' + url + ' and saving it in ' +
new_temp_path)
try:
previous_path = sorted(glob.glob(config.data_folder + '/*.csv'))[-1]
except IndexError:
previous_path = None
try:
urllib.request.urlretrieve(url, new_temp_path)
except urllib.error.URLError as e:
click.echo('Failed to establish an HTTP connection.')
click.echo(e.reason)
sys.exit(1)
except urllib.error.HTTPError as e:
click.echo('Managed to connect but failed with HTTP Error code: ' +
e.code)
click.echo(e.reason)
sys.exit(2)
try:
new_rows = csvio.loadOne(new_temp_path)
if not new_rows[0][0] == "Device:":
click.echo('Managed to connect and fetch data from something, '
'but it was not a CSV from a Comet Web Sensor.')
click.echo((new_rows[0][0]))
sys.exit(3)
# Here we'll try to remove overlapping data points with the last file.
# It get's nasty due to time ajustments done by the sensor.
if previous_path is not None:
previous_rows = csvio.loadOne(previous_path)
data_start = data.get_first_data_point_index(previous_rows)
time_of_newest_data_in_previous = time.datetime_from_row(
previous_rows[data_start], 'second')
filtered_rows = []
for row in new_rows:
if data.not_data_point(row):
continue
time_of_row = time.datetime_from_row(row)
if time_of_newest_data_in_previous < time_of_row:
filtered_rows.append(row)
if not filtered_rows:
if config.verbose:
click.echo('No new rows found in fetched data.')
sys.exit(0)
else:
filtered_rows = new_rows
if config.verbose:
click.echo('Rewriting treated CSV to: ' + new_path)
csvio.writeRows(filtered_rows, new_path)
finally:
os.remove(new_temp_path)
def fetch(config, url):
"""Fetches and stores metrics from Sensor at the URL given."""
new_path = os.path.join(config.data_folder, datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S.csv'))
new_temp_path = new_path + 'temp'
if not url.startswith('http://'):
url = 'http://' + url
url += '/export.csv'
if config.verbose:
click.echo('Fetching data from ' + url + ' and saving it in ' + new_temp_path)
try:
previous_path = sorted(glob.glob(config.data_folder + '/*.csv'))[-1]
except IndexError:
previous_path = None
try:
urllib.request.urlretrieve(url, new_temp_path)
except urllib.error.URLError as e:
click.echo('Failed to establish an HTTP connection.')
click.echo(e.reason)
sys.exit(1)
except urllib.error.HTTPError as e:
click.echo('Managed to connect but failed with HTTP Error code: ' + e.code)
click.echo(e.reason)
sys.exit(2)
try:
new_rows = csvio.loadOne(new_temp_path)
if not new_rows[0][0] == "Device:":
click.echo('Managed to connect and fetch data from something, '
'but it was not a CSV from a Comet Web Sensor.')
click.echo((new_rows[0][0]))
sys.exit(3)
# Here we'll try to remove overlapping data points with the last file.
# It get's nasty due to time ajustments done by the sensor.
if previous_path is not None:
previous_rows = csvio.loadOne(previous_path)
data_start = data.get_first_data_point_index(previous_rows)
time_of_newest_data_in_previous = time.datetime_from_row(previous_rows[data_start], 'second')
filtered_rows = []
for row in new_rows:
if data.not_data_point(row):
continue
time_of_row = time.datetime_from_row(row)
if time_of_newest_data_in_previous < time_of_row:
filtered_rows.append(row)
if not filtered_rows:
if config.verbose:
click.echo('No new rows found in fetched data.')
sys.exit(0)
else:
filtered_rows = new_rows
if config.verbose:
click.echo('Rewriting treated CSV to: ' + new_path)
csvio.writeRows(filtered_rows, new_path)
finally:
os.remove(new_temp_path)
