def get(self, table):
""" clear data """
pagecount.IncrPageCount("export.ClearTable", 1)
verify_dig_sig(self.request, "ClearTable")
table_version = str(utils.get_last_arg(self.request, "tv", ""))
if len(table_version) > 0:
source = get_model(table, "ClearTable")
destination = type(table + table_version, (source, ), {})
else:
destination = get_model(table, "ClearTable")
limit = get_limit(self.request, "ClearTable")
if limit < 1:
limit = 500
elif limit > 500:
limit = 500
query = destination.all()
# cannot delete more than 500 entities in a single call
results = query.fetch(limit)
if results:
self.response.out.write("ClearTable: deleting %d from %s%s\n" %
(len(results), table, table_version))
db.delete(results)
else:
self.response.out.write("ClearTable: %s%s clear complete\n" %
(table, table_version))
def get(self, table):
""" clear data """
pagecount.IncrPageCount("export.ClearTable", 1)
verify_dig_sig(self.request, "ClearTable")
table_version = str(utils.get_last_arg(self.request, "tv", ""))
if len(table_version) > 0:
source = get_model(table, "ClearTable")
destination = type(table + table_version, (source,), {})
else:
destination = get_model(table, "ClearTable")
limit = get_limit(self.request, "ClearTable")
if limit < 1:
limit = 500
elif limit > 500:
limit = 500
query = destination.all()
# cannot delete more than 500 entities in a single call
results = query.fetch(limit)
if results:
self.response.out.write("ClearTable: deleting %d from %s%s\n" % (len(results), table, table_version))
db.delete(results)
else:
self.response.out.write("ClearTable: %s%s clear complete\n" % (table, table_version))
def get(self, table):
""" handle the request to export the table """
pagecount.IncrPageCount("export.ExportTableTSV.attempt", 1)
verify_dig_sig(self.request, "ExportTableTSV")
limit = get_limit(self.request, "ExportTableTSV")
min_key = utils.get_last_arg(self.request, "min_key", "")
model = get_model(table, "ExportTableTSV")
self.response.headers["Content-Type"] = "text/plain"
self.response.out.write(export_table_as_tsv(model, min_key, limit))
pagecount.IncrPageCount("export.ExportTableTSV.success", 1)
def get(self, table):
""" handle the request to export the table """
pagecount.IncrPageCount("export.ExportTableTSV.attempt", 1)
verify_dig_sig(self.request, "ExportTableTSV")
limit = get_limit(self.request, "ExportTableTSV")
min_key = utils.get_last_arg(self.request, "min_key", "")
model = get_model(table, "ExportTableTSV")
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write(export_table_as_tsv(model, min_key, limit))
pagecount.IncrPageCount("export.ExportTableTSV.success", 1)
def verify_dig_sig(request, caller):
"""
require callers pass param &digsig=[string] such that
the hash of the string they pass to us equals QT
"""
digsig = utils.get_last_arg(request, "digsig", "")
# check the passed &digsig with QT...
# and work it if they match
if hashlib.sha512(digsig).hexdigest() != QT:
pagecount.IncrPageCount("export.%s.noDigSig" % caller, 1)
raise Fail("no &digsig")
def verify_dig_sig(request, caller):
"""
require callers pass param &digsig=[string] such that
the hash of the string they pass to us equals QT
"""
digsig = utils.get_last_arg(request, "digsig", "")
# check the passed &digsig with QT...
# and work it if they match
if hashlib.sha512(digsig).hexdigest() != QT:
pagecount.IncrPageCount("export.%s.noDigSig" % caller, 1)
raise Fail("no &digsig")
def get_limit(request, caller):
""" get our limit """
try:
limit = int(utils.get_last_arg(request, "limit", "1000"))
except:
pagecount.IncrPageCount("export.%s.nonIntLimit" % caller, 1)
raise Fail("non integer &limit")
if limit < 1:
limit = 1000
return limit
def get_limit(request, caller):
""" get our limit """
try:
limit = int(utils.get_last_arg(request, "limit", "1000"))
except:
pagecount.IncrPageCount("export.%s.nonIntLimit" % caller, 1)
raise Fail("non integer &limit")
if limit < 1:
limit = 1000
return limit
def get(self, table_from, table_to):
""" handle the request to replicate a table """
pagecount.IncrPageCount("export.TransferTable.attempt", 1)
verify_dig_sig(self.request, "TransferTable")
limit = get_limit(self.request, "TransferTable")
min_key = utils.get_last_arg(self.request, "min_key", "")
if table_from == table_to:
pagecount.IncrPageCount("export.TransferTable.sameTableName", 1)
raise Fail("cannot transfer '%s' to itself" % table_from)
if (table_to[0:len(table_from)] + '_') != (table_from + '_'):
raise Fail("destination must start with '%s_'" % table_from)
verify_table_name(table_to)
# match our type of table
source = get_model(table_from, "TransferTable")
destination = type(table_to, (source, ), {})
if min_key == "":
# a blank key means that we are starting at the top of the table
# so we need to clean out anything that may already be in
# the destination table
while True:
query = destination.all()
# 500 records is the max
results = query.fetch(500)
if results:
db.delete(results)
else:
break
last_key, rows = transfer_table(source, destination, min_key, limit)
self.response.out.write("from %s to %s\nrows\t%d\nlast_key\t%s\n" %
(table_from, table_to, rows, last_key))
pagecount.IncrPageCount("export.TransferTable.success", 1)
def get(self, table_from, table_to):
""" handle the request to replicate a table """
pagecount.IncrPageCount("export.TransferTable.attempt", 1)
verify_dig_sig(self.request, "TransferTable")
limit = get_limit(self.request, "TransferTable")
min_key = utils.get_last_arg(self.request, "min_key", "")
if table_from == table_to:
pagecount.IncrPageCount("export.TransferTable.sameTableName", 1)
raise Fail("cannot transfer '%s' to itself" % table_from)
if (table_to[0 : len(table_from)] + "_") != (table_from + "_"):
raise Fail("destination must start with '%s_'" % table_from)
verify_table_name(table_to)
# match our type of table
source = get_model(table_from, "TransferTable")
destination = type(table_to, (source,), {})
if min_key == "":
# a blank key means that we are starting at the top of the table
# so we need to clean out anything that may already be in
# the destination table
while True:
query = destination.all()
# 500 records is the max
results = query.fetch(500)
if results:
db.delete(results)
else:
break
last_key, rows = transfer_table(source, destination, min_key, limit)
self.response.out.write("from %s to %s\nrows\t%d\nlast_key\t%s\n" % (table_from, table_to, rows, last_key))
pagecount.IncrPageCount("export.TransferTable.success", 1)
def post(self, table):
""" handle the request to populate the table """
pagecount.IncrPageCount("export.PopulateTable.attempt", 1)
verify_dig_sig(self.request, "PopulateTable")
table_version = str(utils.get_last_arg(self.request, "tv", ""))
if len(table_version) > 0:
verify_table_name(table_version)
source = get_model(table, "PopulateTable")
destination = type(table + table_version, (source,), {})
else:
destination = get_model(table, "PopulateTable")
# handle reference properties
def ref_property_UserInfo(field):
rmodel = type("UserInfo" + table_version, (models.UserInfo,), {})
return rmodel.get_by_key_name(field)
def nop(v):
""" this is used for unknown field types """
return v
def str_to_datetime(datetimestring):
""" convert string to a real DateTime object """
# dont need milliseconds here
ar = datetimestring.split(".")
datetime_format = "%Y-%m-%d %H:%M:%S"
return datetime.strptime(ar[0], datetime_format)
def str_to_date(datestring):
""" convert string to a real Date object """
date_format = "%Y-%m-%d"
return datetime.strptime(datestring, date_format).date()
try:
reset = int(utils.get_last_arg(self.request, "reset", "0"))
except:
pagecount.IncrPageCount("export.%s.nonIntLimit" % "PopulateTable", 1)
raise Fail("invalid &reset signal")
if reset == 1:
""" we should only see this with a first batch of records """
logging.info("export.PopulateTable reset signal recvd for %s%s" % (table, table_version))
self.response.out.write("PopulateTable: reset signal recvd, clearing all rows\n")
pagecount.IncrPageCount("export.%s.reset" % "PopulateTable", 1)
while True:
query = destination.all()
# cannot delete more than 500 entities in a single call
# and if there are a lot here we are going to timeout
# anyway but better to try and fail than risk duplicating
results = query.fetch(500)
if results:
logging.info("export.PopulateTable deleting %d from %s%s" % (len(results), table, table_version))
self.response.out.write(
"PopulateTable: deleting %d from %s%s\n" % (len(results), table, table_version)
)
db.delete(results)
else:
logging.info("export.PopulateTable %s%s reset complete" % (table, table_version))
self.response.out.write("PopulateTable: %s%s reset complete\n" % (table, table_version))
break
# one record per line
rows = self.request.get("row").split("\n")
# the first row is a header
header = rows.pop(0).split("\t")
field_type = []
for field in header:
# we are going to want to remember a function for each field type
# but for now all we are doing is initializing the list
field_type.append(None)
limit = get_limit(self.request, "PopulateTable")
logging.info("export.PopulateTable write to %s%s" % (table, table_version))
written = 0
row_number = 0
for row in rows:
row_number += 1
# all of our kind of lines should start "row="
if len(row) > ROW_MARKER_LEN and row[0:ROW_MARKER_LEN] == ROW_MARKER:
fields = row[ROW_MARKER_LEN:].split("\t")
for i, field in enumerate(fields):
if i == 0:
# on the first column (key) we only instantiate our kind of record
try:
# it could be a named key
if not str(field)[0].isdigit():
record = destination(key_name=str(field))
else:
record = destination()
except:
record = destination()
else:
if field is None or len(strip(field)) < 1:
# no field/field value, nothing to do
continue
if field_type[i] != None:
# we think we already know what kind of field this is
try:
# but we could be wrong
setattr(record, header[i], field_type[i](field))
except:
# nothing we can really do about it now except carry on
# and see if we can still make this a good record
logging.warning(
"export.PopulateTable %s = %s not set in row %d of %s%s"
% (header[i], field, row_number, table, table_version)
)
self.response.out.write(
"field %s = %s not set in row %d of %s%s\n"
% (header[i], field, row_number, table, table_version)
)
pass
else:
# on the first row of the file
# we dont know what type of field this is
# but we can try them all until we succeed
# and remember which one worked for subsequent rows
n = 0
while n < MAX_FIELD_TYPES:
if n == FIELD_TYPE_REF:
if table != "UserInterest" or header[i] != "user":
continue
setattr(record, header[i], ref_property_UserInfo(field))
field_type[i] = ref_property_UserInfo
break
elif n == FIELD_TYPE_DATETIME:
try:
setattr(record, header[i], str_to_datetime(field))
field_type[i] = str_to_datetime
break
except:
pass
elif n == FIELD_TYPE_DATE:
try:
setattr(record, header[i], str_to_date(field))
field_type[i] = str_to_date
break
except:
pass
elif n == FIELD_TYPE_STR:
try:
setattr(record, header[i], field)
field_type[i] = str
break
except:
pass
elif n == FIELD_TYPE_BOOL:
try:
setattr(record, header[i], bool(field))
field_type[i] = bool
break
except:
pass
elif n == FIELD_TYPE_INT:
try:
setattr(record, header[i], int(field))
field_type[i] = int
break
except:
pass
elif n == FIELD_TYPE_LONG:
try:
setattr(record, header[i], long(field))
field_type[i] = long
break
except:
pass
elif n == FIELD_TYPE_FLOAT:
try:
setattr(record, header[i], float(field))
field_type[i] = float
break
except:
pass
n += 1
if n >= MAX_FIELD_TYPES:
logging.warning(
"export.PopulateTable unknown field type %s in %s%s"
% (header[i], table, table_version)
)
self.response.out.write(
"unknown field type %s in %s%s\n" % (header[i], table, table_version)
)
field_type[i] = nop
else:
logging.debug("%s is type %d\n" % (header[i], n))
# end-of for each field
try:
# ready to attempt a put
record.put()
written += 1
if written >= limit:
break
except:
logging.error(
"export.PopulateTable put failed at row %d in %s%s" % (row_number, table, table_version)
)
self.response.out.write("put failed at row %d in %s%s\n" % (row_number, table, table_version))
# end-of for each row
logging.info("export.PopulateTable wrote %d rows to %s%s" % (written, table, table_version))
self.response.out.write("wrote %d rows to %s%s\n" % (written, table, table_version))
pagecount.IncrPageCount("export.PopulateTable.success", 1)
def post(self, table):
""" handle the request to populate the table """
pagecount.IncrPageCount("export.PopulateTable.attempt", 1)
verify_dig_sig(self.request, "PopulateTable")
table_version = str(utils.get_last_arg(self.request, "tv", ""))
if len(table_version) > 0:
verify_table_name(table_version)
source = get_model(table, "PopulateTable")
destination = type(table + table_version, (source, ), {})
else:
destination = get_model(table, "PopulateTable")
# handle reference properties
def ref_property_UserInfo(field):
rmodel = type('UserInfo' + table_version, (models.UserInfo, ), {})
return rmodel.get_by_key_name(field)
def nop(v):
""" this is used for unknown field types """
return v
def str_to_datetime(datetimestring):
""" convert string to a real DateTime object """
# dont need milliseconds here
ar = datetimestring.split(".")
datetime_format = "%Y-%m-%d %H:%M:%S"
return datetime.strptime(ar[0], datetime_format)
def str_to_date(datestring):
""" convert string to a real Date object """
date_format = "%Y-%m-%d"
return datetime.strptime(datestring, date_format).date()
try:
reset = int(utils.get_last_arg(self.request, "reset", "0"))
except:
pagecount.IncrPageCount("export.%s.nonIntLimit" % "PopulateTable",
1)
raise Fail("invalid &reset signal")
if reset == 1:
""" we should only see this with a first batch of records """
logging.info("export.PopulateTable reset signal recvd for %s%s" %
(table, table_version))
self.response.out.write(
"PopulateTable: reset signal recvd, clearing all rows\n")
pagecount.IncrPageCount("export.%s.reset" % "PopulateTable", 1)
while True:
query = destination.all()
# cannot delete more than 500 entities in a single call
# and if there are a lot here we are going to timeout
# anyway but better to try and fail than risk duplicating
results = query.fetch(500)
if results:
logging.info("export.PopulateTable deleting %d from %s%s" %
(len(results), table, table_version))
self.response.out.write(
"PopulateTable: deleting %d from %s%s\n" %
(len(results), table, table_version))
db.delete(results)
else:
logging.info("export.PopulateTable %s%s reset complete" %
(table, table_version))
self.response.out.write(
"PopulateTable: %s%s reset complete\n" %
(table, table_version))
break
# one record per line
rows = self.request.get("row").split("\n")
# the first row is a header
header = rows.pop(0).split("\t")
field_type = []
for field in header:
# we are going to want to remember a function for each field type
# but for now all we are doing is initializing the list
field_type.append(None)
limit = get_limit(self.request, "PopulateTable")
logging.info("export.PopulateTable write to %s%s" %
(table, table_version))
written = 0
row_number = 0
for row in rows:
row_number += 1
# all of our kind of lines should start "row="
if len(row
) > ROW_MARKER_LEN and row[0:ROW_MARKER_LEN] == ROW_MARKER:
fields = row[ROW_MARKER_LEN:].split("\t")
for i, field in enumerate(fields):
if i == 0:
# on the first column (key) we only instantiate our kind of record
try:
# it could be a named key
if not str(field)[0].isdigit():
record = destination(key_name=str(field))
else:
record = destination()
except:
record = destination()
else:
if field is None or len(strip(field)) < 1:
# no field/field value, nothing to do
continue
if field_type[i] != None:
# we think we already know what kind of field this is
try:
# but we could be wrong
setattr(record, header[i],
field_type[i](field))
except:
# nothing we can really do about it now except carry on
# and see if we can still make this a good record
logging.warning(
"export.PopulateTable %s = %s not set in row %d of %s%s"
% (header[i], field, row_number, table,
table_version))
self.response.out.write(
"field %s = %s not set in row %d of %s%s\n"
% (header[i], field, row_number, table,
table_version))
pass
else:
# on the first row of the file
# we dont know what type of field this is
# but we can try them all until we succeed
# and remember which one worked for subsequent rows
n = 0
while n < MAX_FIELD_TYPES:
if n == FIELD_TYPE_REF:
if table != "UserInterest" or header[
i] != "user":
continue
setattr(record, header[i],
ref_property_UserInfo(field))
field_type[i] = ref_property_UserInfo
break
elif n == FIELD_TYPE_DATETIME:
try:
setattr(record, header[i],
str_to_datetime(field))
field_type[i] = str_to_datetime
break
except:
pass
elif n == FIELD_TYPE_DATE:
try:
setattr(record, header[i],
str_to_date(field))
field_type[i] = str_to_date
break
except:
pass
elif n == FIELD_TYPE_STR:
try:
setattr(record, header[i], field)
field_type[i] = str
break
except:
pass
elif n == FIELD_TYPE_BOOL:
try:
setattr(record, header[i], bool(field))
field_type[i] = bool
break
except:
pass
elif n == FIELD_TYPE_INT:
try:
setattr(record, header[i], int(field))
field_type[i] = int
break
except:
pass
elif n == FIELD_TYPE_LONG:
try:
setattr(record, header[i], long(field))
field_type[i] = long
break
except:
pass
elif n == FIELD_TYPE_FLOAT:
try:
setattr(record, header[i],
float(field))
field_type[i] = float
break
except:
pass
n += 1
if n >= MAX_FIELD_TYPES:
logging.warning(
"export.PopulateTable unknown field type %s in %s%s"
% (header[i], table, table_version))
self.response.out.write(
"unknown field type %s in %s%s\n" %
(header[i], table, table_version))
field_type[i] = nop
else:
logging.debug("%s is type %d\n" %
(header[i], n))
# end-of for each field
try:
# ready to attempt a put
record.put()
written += 1
if written >= limit:
break
except:
logging.error(
"export.PopulateTable put failed at row %d in %s%s" %
(row_number, table, table_version))
self.response.out.write("put failed at row %d in %s%s\n" %
(row_number, table, table_version))
# end-of for each row
logging.info("export.PopulateTable wrote %d rows to %s%s" %
(written, table, table_version))
self.response.out.write("wrote %d rows to %s%s\n" %
(written, table, table_version))
pagecount.IncrPageCount("export.PopulateTable.success", 1)
