def getoscdiff(timebase, cat1, cat2, cat3):
timenow = int(time.time())
epochts = int(time.mktime(settings.REPLICATE_EPOCH.timetuple()))
if timebase == "minute":
pageStep = 60000000
if timebase == "hour":
pageStep = 60000000 * 60
if timebase == "day":
pageStep = 60000000 * 60 * 24
pageStep2 = pageStep // 1000
pageStep3 = pageStep2 // 1000
pageStartTimestamp = (
int(cat1) - settings.REPLICATE_OFFSET
) * pageStep + int(cat2) * pageStep2 + int(cat3) * pageStep3 + epochts
elapsedInPage = timenow - pageStartTimestamp
if elapsedInPage < 0:
return HttpResponseNotFound("Page does not exist")
t = p.GetTransaction("EXCLUSIVE")
osmc = pgmap.OsmChange()
t.GetReplicateDiff(pageStartTimestamp - pageStep3, pageStartTimestamp,
osmc)
sio = io.BytesIO()
pgmap.SaveToOsmChangeXml(osmc, False, pgmap.CPyOutbuf(sio))
return sio.getvalue()
def test_delete_node_diff(self):
ts = time.time() - 120
ts2 = time.time() - 60
node = qmt.create_node(self.user.id, self.user.username, timestamp = ts)
ok = qmt.delete_object(node, self.user, timestamp = ts2)
self.assertEqual(ok, True)
cat = rv.TimestampToPath(int(ts2), "minute")
anonClient = Client()
response = anonClient.get(reverse('replicate:diffdata', args=['minute', str(cat[0]), str(cat[1]), str(cat[2])]))
if response.status_code != 200:
print (response.content)
self.assertEqual(response.status_code, 200)
osc = pgmap.OsmChange()
pgmap.LoadFromOsmChangeXml(response.content.decode("utf-8"), osc)
oscIndex = CreateIndexOsmChange(osc)
self.assertEqual(node.objId in oscIndex["node"], True)
found = False
for action, obj in oscIndex["node"][node.objId]:
print (action, obj.objId, obj.metaData.version)
if action == "delete":
found = True
break
self.assertEqual(found, True)
def parse(self, stream, media_type, parser_context):
data = pgmap.OsmChange()
dec = pgmap.OsmChangeXmlDecodeString()
dec.output = data
pageSize = 100000
while True:
inputXml = stream.read(pageSize)
if len(inputXml) == 0:
break
dec.DecodeSubString(inputXml.decode("UTF-8"), len(inputXml), False)
dec.DecodeSubString("", 0, True)
if not dec.parseCompletedOk:
raise ParseError(detail=dec.errString)
return data
def customdiff(request):
#This is a non-standard (pycrocosm specific) API call to get diffs of custom time ranges.
startTsArg = request.GET.get('start', None) #Normally ISO 8601
endTsArg = request.GET.get('end', None) #Normally ISO 8601
compress = request.GET.get('compress', 'no')
if startTsArg is None:
return HttpResponseBadRequest("start argument not set")
if endTsArg is None:
return HttpResponseBadRequest("end argument not set")
startTs = parse_datetime(startTsArg)
if startTs is None:
startTs = parse_date(startTsArg)
endTs = parse_datetime(endTsArg)
if endTs is None:
endTs = parse_date(endTsArg)
if startTs is None:
return HttpResponseBadRequest(
"start argument not understood (should be ISO 8601 date or datetime)"
)
if endTs is None:
return HttpResponseBadRequest(
"end argument not understood (should be ISO 8601 date or datetime)"
)
if endTs < startTs:
return HttpResponseBadRequest("end cannot be before start")
t = p.GetTransaction("EXCLUSIVE")
osmc = pgmap.OsmChange()
t.GetReplicateDiff(int(time.mktime(startTs.timetuple())),
int(time.mktime(endTs.timetuple())), osmc)
sio = io.BytesIO()
pgmap.SaveToOsmChangeXml(osmc, False, pgmap.CPyOutbuf(sio))
if compress == 'no':
return HttpResponse(sio.getvalue(), content_type='text/xml')
if compress == 'gz':
comp = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED,
zlib.MAX_WBITS | 16)
gzip_data = comp.compress(sio.getvalue()) + comp.flush()
return HttpResponse(gzip_data, content_type='application/x-gzip')
return HttpResponseBadRequest("compression argument not understood")
def download(request, changesetId):
t = p.GetTransaction("ACCESS SHARE")
osmChange = pgmap.OsmChange()
errStr = pgmap.PgMapError()
ret = t.GetChangesetOsmChange(int(changesetId), osmChange, errStr)
if ret == -1:
return HttpResponseNotFound("Changeset not found")
if ret == 0:
return HttpResponseServerError(errStr.errStr)
t.Commit()
#print (changesetData.data.empty())
sio = io.BytesIO()
outBufWrapped = pgmap.CPyOutbuf(sio)
pgmap.SaveToOsmChangeXml(osmChange, outBufWrapped)
return HttpResponse(sio.getvalue(), content_type='text/xml')
def test_create_node_diff(self):
ts = time.time() - 60
node = qmt.create_node(self.user.id, self.user.username, timestamp = ts)
cat = rv.TimestampToPath(node.metaData.timestamp, "minute")
#Check minutely diff
anonClient = Client()
response = anonClient.get(reverse('replicate:diffdata', args=['minute', str(cat[0]), str(cat[1]), str(cat[2])]))
if response.status_code != 200:
print (response.content)
self.assertEqual(response.status_code, 200)
osc = pgmap.OsmChange()
pgmap.LoadFromOsmChangeXml(response.content.decode("utf-8"), osc)
oscIndex = CreateIndexOsmChange(osc)
self.assertEqual(node.objId in oscIndex["node"], True)
self.assertEqual(oscIndex["node"][node.objId][0][0], "create")
def test_create_node_diff_custom_range(self):
ts = time.time() - 60
node = qmt.create_node(self.user.id, self.user.username, timestamp = ts)
#Check custom time range diff
anonClient = Client()
startTs = datetime.datetime.utcfromtimestamp(ts-5).isoformat()
endTs = datetime.datetime.utcfromtimestamp(ts+5).isoformat()
response = anonClient.get(reverse('replicate:customdiff')+'?start={}&end={}'.format(startTs, endTs))
if response.status_code != 200:
print (response.content)
self.assertEqual(response.status_code, 200)
osc = pgmap.OsmChange()
pgmap.LoadFromOsmChangeXml(response.content.decode("utf-8"), osc)
oscIndex = CreateIndexOsmChange(osc)
self.assertEqual(node.objId in oscIndex["node"], True)
self.assertEqual(oscIndex["node"][node.objId][0][0], "create")
def test_create_relation_diff(self):
ts = time.time() - 60
node1 = qmt.create_node(self.user.id, self.user.username, timestamp = ts)
node2 = qmt.create_node(self.user.id, self.user.username, timestamp = ts)
refs = [("node", node1.objId, "foo"), ("node", node2.objId, "bar")]
relation = qmt.create_relation(self.user.id, self.user.username, refs, timestamp = ts)
cat = rv.TimestampToPath(relation.metaData.timestamp, "minute")
anonClient = Client()
response = anonClient.get(reverse('replicate:diffdata', args=['minute', str(cat[0]), str(cat[1]), str(cat[2])]))
if response.status_code != 200:
print (response.content)
self.assertEqual(response.status_code, 200)
osc = pgmap.OsmChange()
pgmap.LoadFromOsmChangeXml(response.content.decode("utf-8"), osc)
oscIndex = CreateIndexOsmChange(osc)
self.assertEqual(relation.objId in oscIndex["relation"], True)
self.assertEqual(oscIndex["relation"][relation.objId][0][0], "create")