def send_data(self):
if not self._archive:
self.log_warn('No archive previously created. We should not have been called.')
return
cfg_server = self._config[_CFG_PROPS.SERVER]
url = self._config[_CFG_PROPS.API_URLS][_CFG_PROPS.DATA_UPLOAD] % {
'host': cfg_server[_CFG_PROPS.HOST],
'site': self._site_code
}
auth = cfg_server[_CFG_PROPS.AUTH]
with file(self._archive, 'rb') as archive:
self.log_info('uploading file %s using URL %s', self._archive, url)
resp = requests.post(
url,
files={
'zip': archive
},
auth=(auth[_CFG_PROPS.LOGIN], auth[_CFG_PROPS.PASSWORD])
)
self.log_info('%s - %s', resp, resp.text)
if resp.ok:
resp_data = json.loads(resp.text)
job_id = resp_data['jobID']
self.log_info('upload successful (%s) - job id=%s' % (resp_data['message'], job_id))
# nothing else to be done : data have been transmitted
# if anomalies are detected during the processing, they will be notified to the
# site manager, and there is no need to activate the backlog retry mechanism here,
# since it will not fix the anomaly but just reproduce it again and again
# Anyway, for record's sake, we go one step further by monitoring the job completion status
# and log the result
# add the job id to the persistent queue
queue = PendingJobsQueue()
queue.append(job_id)
else:
try:
msg = 'failed : %d - %s (%s)' % (resp.status_code, resp.reason, resp.text)
except ValueError:
msg = 'unexpected server error : %d - %s' %(resp.status_code, resp.reason)
self.log_error(msg)
raise pycstbox.export.ExportError(msg)
def run(self):
site_code = self._cfg.site_code
period = int(self._cfg.status_monitoring_period)
auth = (self._cfg.login, self._cfg.password)
query = self._cfg.job_status_url
self._log.info('started (site_code=%s period=%d secs)', site_code, period)
last_check = 0
while True:
now = time.time()
if now - last_check >= period:
queue = PendingJobsQueue()
for job_id in queue.items():
if self._debug:
self._log.debug('requesting status of site/job %s/%s', self._cfg.site_code, job_id)
resp = requests.get(url=query % (site_code, job_id), auth=auth)
if resp.ok:
if self._debug:
self._log.debug('got reply : %s', resp.text)
reply = json.loads(resp.text)
completion_code = reply['code']
if completion_code == 0: # completed
# log it and remove the job from the queue
log.info('job %s completed ok', job_id)
queue.remove(job_id)
elif completion_code < 0:
# solid error => log it and remove the job from the queue
try:
code_msg = reply['status']
except KeyError:
code_msg = "unknown code"
log.error('job %s failed with code %d (%s)', job_id, completion_code, code_msg)
queue.remove(job_id)
# otherwise the job is still pending. Just leave it a is
else:
log.error("server replied with : %d - %s", resp.status_code, resp.reason)
last_check = now
if self._terminated:
self._log.info('terminate request detected')
break
time.sleep(self.TERMINATE_CHECK_PERIOD)
self._log.info('worker thread terminated')