def testGetChannelByName(self):
db = getDbObject(DB)
obj_id = db[COLLECTION].save({NAME: "test_GT_1288"})
with self.assertRaises(ChannelDoesNotExist):
getChannelByName("testservice", "111")
obj = getChannelByName("testservice", "test_GT_1288")
self.assertEqual(obj["_id"], obj_id)
def testGetChannelByName(self):
db = getDbObject(DB)
obj_id = db[COLLECTION].save({NAME: 'test_GT_1288'})
with self.assertRaises(ChannelDoesNotExist):
getChannelByName('testservice', '111')
obj = getChannelByName('testservice', 'test_GT_1288')
self.assertEqual(obj['_id'], obj_id)
def post(self, serviceName):
listArgs = ChannelsListResourceParser.parsePostParameters()
try:
getChannelByName(serviceName, listArgs.get(NAME, None))
except ChannelDoesNotExist:
return addChannel(
listArgs.get(
NAME, None), listArgs.get(
JSON, None), STUB, serviceName)
def post(self, serviceName):
importDataDict = parserClass.parsePostParameters()
channelName = importDataDict.get('channelName')
getServiceIdByName(serviceName)
getChannelByName(serviceName, channelName)
job = jobClass(importFunction, importDataDict.get('channelName'),
importDataDict.get('openDataUrl'), importDataDict,
serviceName)
return JobManager.startJob(job)
def geocoderImport(self, channelName, serviceName):
# Checking for service existence
getServiceIdByName(serviceName)
# Getting channel id
channel = getChannelByName(serviceName, channelName)
channel_id = channel[ID]
# Initialising requester and parser
greqv = GeonamesRequestSender()
gpars = GeocoderResponseParser()
# Getting points collection
points_coll = getDbObject(serviceName)[POINTS_COLL]
# Getting first slice of <= 5000 points
proc_step = 0
point_block = list(points_coll.find({
CHANNEL_ID: channel_id,
JSON + '.' + ADDRESS: {"$exists": True}
}).skip(proc_step).limit(PROC_BLOCK))
login = PluginConfigReader(PLUGIN_NAME).getConfigContent()[
SECTION_GEOCODING][GEONAMES_LOGIN]
while point_block:
current_size = len(point_block)
addresses = [point[JSON][ADDRESS] for point in point_block]
ids = [point[ID] for point in point_block]
string_addresses = greqv.requestCoordinates(
addresses,
login,
json_list_to_list_of_strings)
json_coords = gpars.parseList(string_addresses)
for i in range(0, current_size):
# If no coords was found - we change nothing
if json_coords[i] is None:
continue
points_coll.update(
{ID: ids[i]},
{'$set': {'location.coordinates': json_coords[i]}}
)
# Getting next slice of <= 5000 points
proc_step += PROC_BLOCK
point_block = list(
points_coll.find().skip(proc_step).limit(PROC_BLOCK))
# Task is done
self.done = True
# And thread is done too after return
return []
def performImportActions(
odLoaderClass,
odParserClass,
odToPointTranslatorClass,
odToPointsLoaderClass,
channelName,
openDataUrl,
importDataDict,
serviceName):
# uncomment in case of https://geo2tag.atlassian.net/browse/GT-1505
'''if issubclassof(odLoaderClass, AbstractOpenDataLoader):
if issubclassof(odParserClass, )
if issubclassof(odToPointTranslatorClass, )
if issubclassof(odToPointsLoaderClass, AbstractOpenDataToPointsLoader)
'''
channelId = getChannelByName(serviceName, channelName)['_id']
version = datetime.now()
loader = odLoaderClass(openDataUrl)
openData = loader.load()
parser = odParserClass(openData)
objects = parser.parse()
points = []
for objectRepresentation in objects:
translator = odToPointTranslatorClass(
importDataDict,
objectRepresentation,
version,
openDataUrl,
channelId)
points.append(translator.getPoint())
pointsLoader = odToPointsLoaderClass(serviceName, points)
pointsLoader.loadPoints()