def apiAdd(clientID, sourceID, data, schemaOverRide):
if isinstance(data, dict):
dataList = [data]
else:
dataList = data
toInsert = []
for item in dataList:
item = ProcessData.flatten(data=item)
item = ProcessData.enrich(data=item, schemaOverRide=schemaOverRide)
date = ProcessData.getNow()
toInsert.append({'created': date, 'data': item})
SourceHandler.add(clientID=clientID, sourceID=sourceID, data=toInsert)
def add(self, data):
Add.apiAdd.delay(clientID=str(self.client.id),
sourceID=str(self.id),
data=data,
schemaOverRide=SourceHandler.getSchema(
clientID=str(self.client.id),
sourceID=str(self.id)))
def dict(self):
response = getDict(self)
del response['client']
del response['credentials']
# del response['schemaOverRide']
response['schema'] = SourceHandler.getSchema(clientID=str(
self.client.id),
sourceID=str(self.id))
# response['schema'].update(self.schemaOverRide)
return response
def overRideSchema(self, field, toType):
currentSchema = SourceHandler.getSchema(clientID=str(self.client.id),
sourceID=str(self.id))
currentType = currentSchema.get(field, None)
if self.schemaChanges.get(field, None) is None:
self.schemaChanges[field] = {}
if currentType is None: return
self.schemaChanges[field] = {'from': currentType, 'to': toType}
updateSchema.delay(clientID=str(self.client.id),
sourceID=str(self.id),
fromType=currentType,
key=field,
toType=toType)
self.save()
def __init__(self,clientID,query,sourceID,utcOffset="+00"):
start = query.get('start')
end = query.get('end')
conditions = query.get('conditions')
increment = query.get('increment')
groupBy = query.get('groupBy')
operation = {'source': sourceID,'field': query['field'],'type': query['type']}
assert type(clientID) is str
assert increment in list(INCREMENTS.values())
assert end == 'now' or type(end) is datetime.datetime
assert type(start) is int or type(end) is datetime.datetime
assert operation.get('type',None) in ['sum','count']
assert type(operation.get('source',None)) is str
assert type(operation.get('field',None)) is str or operation.get('type',None) == 'count'
assert type(groupBy) is str or type(groupBy) is not None
for item in conditions:
assert type(item['field']) is str
assert item['operation'] in ALLOWED_OPERATIONS
assert type(item['constant']) is str
for condition in conditions:
condition['operation'] = "$" + condition.get('operation','')
datePath = query.get('yAxis')
if datePath == '$created$':
datePath = "created"
else:
datePath = "data." + datePath
groupsColors = query.get('groupColors',[])
dates = self.__getDates(start=start,end=end,increment=increment)
start = dates['start']
end = dates['end']
start,end,increment,availableIncrements=getResolution(start=start,end=end,increment=increment)
self.__pipline = []
self.__setConditions(conditions=conditions)
self.__setTimeInteval(start=start,end=end,datePath=datePath)
self.__setOperation(operation=operation,groupBy=groupBy,increment=increment,datePath=datePath,utcOffset=utcOffset)
source = SourceHandler.getSource(clientID=clientID,sourceID=operation['source'])
self.__query(source=source)
self.__normalize(start=start,end=end,increment=increment,groupsColors=groupsColors,utcOffset=utcOffset)
self.__end = str(end)
self.__start = str(start)
self.__increment = str(increment)
self.__availableIncrements = availableIncrements
def cacheSchema(self):
SourceHandler.cacheSchema(clientID=str(self.client.id),
sourceID=str(self.id))
def updateSchema(clientID, sourceID, key, fromType, toType):
SourceHandler.editConvertTypes(clientID=clientID,
sourceID=sourceID,
key=key,
fromType=fromType,
toType=toType)
def remove(clientID, sourceID):
SourceHandler.delete(clientID, sourceID)