def __loadModule(self, modules):
if modules is None:
logger.Info('No module list found')
return None
loaded_modules = []
for name in modules:
loaded_module = importlib.import_module('glouton.modules.' +
name.lower())
module = getattr(loaded_module, name)
loaded_modules.append(module(self.__cmd.working_dir))
logger.Info('module : ' + name + ' loaded')
return loaded_modules
def scan(self):
params = self.__url_params
page = 1
job_string = 'job ' + self.__job_number
while True:
r = self.__client.get_from_base(self.PATH, params)
if r.status_code != 200:
break
logger.Info(job_string + ' scanning page...' + params['page'])
self.__read_page(r.json(), self.__cmd.start_date,
self.__cmd.end_date)
page += 1
params['page'] = str(page)
logger.Info(job_string + ' terminated')
def loadWaterfallModules(self):
logger.Info('Waterfall module(s) loading :')
self.__cmd.waterfall_modules = self.__getModulesFromConfig(
self.__cmd.waterfall_modules, 'WATERFALL')
self.__cmd.waterfall_modules = self.__getModulesFromConfig(
self.__cmd.waterfall_modules, 'FORALL')
return self.__loadModule(self.__cmd.waterfall_modules)
def loadPayloadModules(self):
logger.Info('Payload module(s) loading :')
self.__cmd.payload_modules = self.__getModulesFromConfig(
self.__cmd.payload_modules, 'PAYLOAD')
self.__cmd.payload_modules = self.__getModulesFromConfig(
self.__cmd.payload_modules, 'FORALL')
return self.__loadModule(self.__cmd.payload_modules)
def loadDemoddataModules(self):
logger.Info('Demoddata module(s) loading :')
self.__cmd.demoddata_modules = self.__getModulesFromConfig(
self.__cmd.demoddata_modules, 'DEMODDATA')
self.__cmd.demoddata_modules = self.__getModulesFromConfig(
self.__cmd.demoddata_modules, 'FORALL')
return self.__loadModule(self.__cmd.demoddata_modules)
def loadWaterfallModules(self, waterfall_modules, when):
logger.Info('(' + when + ') Waterfall module(s) loading :')
waterfall_modules = self.__getModulesFromConfig(
waterfall_modules, when, 'WATERFALL')
waterfall_modules = self.__getModulesFromConfig(
waterfall_modules, when, 'FOR_ALL_OBSERVATION')
return self.__loadModule(waterfall_modules)
def loadArchiveModules(self, archive_modules, when):
logger.Info('(' + when + ') Archive module(s) loading :')
archive_modules = self.__getModulesFromConfig(
archive_modules, when, 'ARCHIVE')
archive_modules = self.__getModulesFromConfig(
archive_modules, when, 'FOR_ALL_OBSERVATION')
return self.__loadModule(archive_modules)
def loadDemoddataModules(self, demoddata_modules, when):
logger.Info('(' + when + ') Demoddata module(s) loading :')
demoddata_modules = self.__getModulesFromConfig(
demoddata_modules, when, 'DEMODDATA')
demoddata_modules = self.__getModulesFromConfig(
demoddata_modules, when, 'FOR_ALL_OBSERVATION')
return self.__loadModule(demoddata_modules)
def download(self):
frame = self.telemetry[self.__json_id]
if not frame:
logger.Info('no frame found for the telemetry of ' +
self.telemetry['timestamp'])
return
fileHelper.create_dir_if_not_exist(self.full_path)
self.runModulesAfterDownload(frame)
def download(self):
url = self.observation[self.__json_id]
if not url:
logger.Info('no archive found for the observation ' +
str(self.observation['id']) + ' of ' +
self.observation['start'])
return
fileHelper.create_dir_if_not_exist(self.full_path)
file_name = ntpath.basename(url)
full_path_file = self.full_path + os.path.sep + file_name
if os.path.exists(full_path_file):
logger.Warning('pass ' + file_name + '... file already exist')
return
r = self.client.get(url)
if r.status_code == 200:
logger.Info('downloading archive...' + file_name)
with open(full_path_file, "wb") as file:
file.write(r.content)
self.runModulesAfterDownload(file_name)
def extract(self):
params = self.__create_request_params()
page = 1
while True:
r = self.__client.get_from_base(self.OBSERVATION_URL, params)
if r.status_code != 200:
break
logger.Info('scanning page...' + params['page'])
self.__read_page(r.json(), self.__cmd.start_date,
self.__cmd.end_date)
page += 1
params['page'] = str(page)
print('\ndownloading started (Ctrl + C to stop)...\t~( ^o^)~')
self.__create_workers_and_wait()
def loadFrameModules(self, frame_modules, when):
logger.Info('(' + when + ') Frame module(s) loading :')
frame_modules = self.__getModulesFromConfig(
frame_modules, when, 'FRAME')
return self.__loadModule(frame_modules)