def send_data(self): if not self._archive: self.log_warn('No archive previously created. We should not have been called.') return cfg_server = self._config[_CFG_PROPS.SERVER] url = self._config[_CFG_PROPS.API_URLS][_CFG_PROPS.DATA_UPLOAD] % { 'host': cfg_server[_CFG_PROPS.HOST], 'site': self._site_code } auth = cfg_server[_CFG_PROPS.AUTH] with file(self._archive, 'rb') as archive: self.log_info('uploading file %s using URL %s', self._archive, url) resp = requests.post( url, files={ 'zip': archive }, auth=(auth[_CFG_PROPS.LOGIN], auth[_CFG_PROPS.PASSWORD]) ) self.log_info('%s - %s', resp, resp.text) if resp.ok: resp_data = json.loads(resp.text) job_id = resp_data['jobID'] self.log_info('upload successful (%s) - job id=%s' % (resp_data['message'], job_id)) # nothing else to be done : data have been transmitted # if anomalies are detected during the processing, they will be notified to the # site manager, and there is no need to activate the backlog retry mechanism here, # since it will not fix the anomaly but just reproduce it again and again # Anyway, for record's sake, we go one step further by monitoring the job completion status # and log the result # add the job id to the persistent queue queue = PendingJobsQueue() queue.append(job_id) else: try: msg = 'failed : %d - %s (%s)' % (resp.status_code, resp.reason, resp.text) except ValueError: msg = 'unexpected server error : %d - %s' %(resp.status_code, resp.reason) self.log_error(msg) raise pycstbox.export.ExportError(msg)
def run(self): site_code = self._cfg.site_code period = int(self._cfg.status_monitoring_period) auth = (self._cfg.login, self._cfg.password) query = self._cfg.job_status_url self._log.info('started (site_code=%s period=%d secs)', site_code, period) last_check = 0 while True: now = time.time() if now - last_check >= period: queue = PendingJobsQueue() for job_id in queue.items(): if self._debug: self._log.debug('requesting status of site/job %s/%s', self._cfg.site_code, job_id) resp = requests.get(url=query % (site_code, job_id), auth=auth) if resp.ok: if self._debug: self._log.debug('got reply : %s', resp.text) reply = json.loads(resp.text) completion_code = reply['code'] if completion_code == 0: # completed # log it and remove the job from the queue log.info('job %s completed ok', job_id) queue.remove(job_id) elif completion_code < 0: # solid error => log it and remove the job from the queue try: code_msg = reply['status'] except KeyError: code_msg = "unknown code" log.error('job %s failed with code %d (%s)', job_id, completion_code, code_msg) queue.remove(job_id) # otherwise the job is still pending. Just leave it a is else: log.error("server replied with : %d - %s", resp.status_code, resp.reason) last_check = now if self._terminated: self._log.info('terminate request detected') break time.sleep(self.TERMINATE_CHECK_PERIOD) self._log.info('worker thread terminated')