class Exporter:
def __init__(self):
self.client = Client(API_KEY, API_KEY, API_SECRET)
self.series_filename = os.path.join(EXPORT_DIR, "series_info")
if not os.path.exists(EXPORT_DIR):
print "Making export directory"
os.makedirs(EXPORT_DIR)
def export_metadata(self):
if os.path.isfile(self.series_filename):
print "series_info exists, skipping series discovery"
return
print "Exporting series metadata"
all_series = self.client.list_series()
with open(self.series_filename, 'w') as outfile:
for series in all_series.data:
line = self.series_to_string(series) + "\n"
outfile.write(line.encode("utf-8"))
def series_to_string(self, series):
local_id = uuid.uuid4()
j = {"uuid": str(local_id)} # Generate series UUID since keys
for p in series.properties: # could be inconvenient filenames
j[p] = getattr(series, p)
return json.dumps(j, ensure_ascii=False)
def export_all_series(self):
if not os.path.isfile(self.series_filename):
print "ERROR: No series_info file found, can't export series data"
return
with open(self.series_filename, 'r') as series_list:
for text in series_list:
series = json.loads(text)
self.export_single_series(series.get('key'),
series.get('uuid'))
def export_single_series(self, key, uuid):
filename = os.path.join(EXPORT_DIR, uuid + ".csv")
if os.path.isfile(filename):
print "Data file exists for series " + key + ", skipping"
return
print "Exporting series " + key + " to " + filename
response = self.client.read_data(key=key.encode('utf-8'),
start=START_DATE,
end=END_DATE)
with open(filename, 'w') as outfile:
for dp in response.data:
line = dp.t.isoformat() + "," + str(dp.v) + "\n"
outfile.write(line)
class Exporter:
def __init__(self):
self.client = Client(API_KEY, API_KEY, API_SECRET)
self.series_filename = os.path.join(EXPORT_DIR, "series_info")
if not os.path.exists(EXPORT_DIR):
print "Making export directory"
os.makedirs(EXPORT_DIR)
def export_metadata(self):
if os.path.isfile(self.series_filename):
print "series_info exists, skipping series discovery"
return
print "Exporting series metadata"
all_series = self.client.list_series()
with open(self.series_filename, 'w') as outfile:
for series in all_series:
line = self.series_to_string(series) + "\n"
outfile.write(line.encode("utf-8"))
def series_to_string(self, series):
local_id = uuid.uuid4()
j = {"uuid": str(local_id)} # Generate series UUID since keys
for p in series.properties: # could be inconvenient filenames
j[p] = getattr(series, p)
return json.dumps(j, ensure_ascii=False)
def export_all_series(self):
if not os.path.isfile(self.series_filename):
print "ERROR: No series_info file found, can't export series data"
return
with open(self.series_filename, 'r') as series_list:
for text in series_list:
series = json.loads(text)
self.export_single_series(series.get('key'),
series.get('uuid'))
def export_single_series(self, key, uuid):
filename = os.path.join(EXPORT_DIR, uuid + ".csv")
if os.path.isfile(filename):
print "Data file exists for series " + key + ", skipping"
return
print "Exporting series " + key + " to " + filename
response = self.client.read_data(key=key.encode('utf-8'), start=START_DATE, end=END_DATE)
with open(filename, 'w') as outfile:
for dp in response:
line = dp.t.isoformat() + "," + str(dp.v) + "\n"
outfile.write(line)
class Migrator:
def __init__(self, scheme, create_devices=True,
write_data=True,
start_date="2000-01-01T00:00:00Z",
end_date="2014-12-31T00:00:00Z",
pool_size=3):
self.scheme = scheme
self.create_devices = create_devices
self.should_write_data = write_data
self.start_date = start_date
self.end_date = end_date
self.tdb = TDBClient(scheme.db_key, scheme.db_key,
scheme.db_secret,
base_url=scheme.db_baseurl)
iq_endpoint = HTTPEndpoint(scheme.iq_baseurl,
scheme.iq_key,
scheme.iq_secret)
self.tiq = TIQClient(iq_endpoint)
self.queue = JoinableQueue()
self.lock = Lock()
self.dp_count = 0
self.req_count = 0
self.dp_reset = time.time()
for i in range(pool_size):
gevent.spawn(self.worker)
def worker(self):
while True:
series = self.queue.get()
try:
self.migrate_series(series)
finally:
self.queue.task_done()
def migrate_all_series(self, start_key="", limit=None):
start_time = time.time()
(keys, tags, attrs) = self.scheme.identity_series_filter()
series_set = self.tdb.list_series(keys, tags, attrs)
# Keep our own state of whether we passed the resume point, so we don't
# need to assume client and server sort strings the same.
found_first_series = False
series_count = 0
for series in series_set:
if not found_first_series and series.key < start_key:
continue
else:
found_first_series = True
if limit and series_count >= limit:
print("Reached limit of %d devices, stopping." % (limit))
break
if self.scheme.identity_series_client_filter(series):
# If the series looks like an identity series,
# queue it to be processed by the threadpool
self.queue.put(series)
series_count += 1
self.queue.join()
end_time = time.time()
print("Exporting {} devices took {} seconds".format(series_count, end_time - start_time))
def migrate_series(self, series):
print(" Beginning to migrate series: %s" % (series.key))
error = False
try:
if self.create_devices:
error = self.create_device(series)
if self.should_write_data and not error:
error = self.write_data(series)
except Exception, e:
logging.exception(e)
error = True
if not error:
print("COMPLETED migrating for series %s" % (series.key))
else:
print("ERROR migrating series %s" % (series.key))
class Migrator:
def __init__(self,
scheme,
create_devices=True,
write_data=True,
start_date="2000-01-01T00:00:00Z",
end_date="2014-12-31T00:00:00Z",
pool_size=3):
self.scheme = scheme
self.create_devices = create_devices
self.should_write_data = write_data
self.start_date = start_date
self.end_date = end_date
self.tdb = TDBClient(scheme.db_key,
scheme.db_key,
scheme.db_secret,
base_url=scheme.db_baseurl)
iq_endpoint = HTTPEndpoint(scheme.iq_baseurl, scheme.iq_key,
scheme.iq_secret)
self.tiq = TIQClient(iq_endpoint)
self.queue = JoinableQueue()
self.lock = Lock()
self.dp_count = 0
self.req_count = 0
self.dp_reset = time.time()
for i in range(pool_size):
gevent.spawn(self.worker)
def worker(self):
while True:
series = self.queue.get()
try:
self.migrate_series(series)
finally:
self.queue.task_done()
def migrate_all_series(self, start_key="", limit=None):
start_time = time.time()
(keys, tags, attrs) = self.scheme.identity_series_filter()
series_set = self.tdb.list_series(keys, tags, attrs)
# Keep our own state of whether we passed the resume point, so we don't
# need to assume client and server sort strings the same.
found_first_series = False
series_count = 0
for series in series_set:
if not found_first_series and series.key < start_key:
continue
else:
found_first_series = True
if limit and series_count >= limit:
print("Reached limit of %d devices, stopping." % (limit))
break
if self.scheme.identity_series_client_filter(series):
# If the series looks like an identity series,
# queue it to be processed by the threadpool
self.queue.put(series)
series_count += 1
self.queue.join()
end_time = time.time()
print("Exporting {} devices took {} seconds".format(
series_count, end_time - start_time))
def migrate_series(self, series):
print(" Beginning to migrate series: %s" % (series.key))
error = False
try:
if self.create_devices:
error = self.create_device(series)
if self.should_write_data and not error:
error = self.write_data(series)
except Exception, e:
logging.exception(e)
error = True
if not error:
print("COMPLETED migrating for series %s" % (series.key))
else:
print("ERROR migrating series %s" % (series.key))
