def get_divide(a, b): try: r = a / b return r except Exception as e: logger.error("calc error, {}".format(e)) return None
def join_staging_task(self, task_id): try: staging_task = get_task(task_id) logger.debug('staging task for %s is: %s' % (task_id, staging_task)) # BUG: FIXME: why is this a list and not a file, we do use uselist=False sf = staging_task.stagable_file[0] logger.info('staging task %s completed, file is %s' % (task_id, sf.name)) handle_staging_result(staging_task, sf) tasks.session.commit() except Exception, e: logger.error('join_staging_task(task_id=%s) unexpectedly failed (task ' 'id %s): %s, retrying in 60 s...' % (task_id, join_staging_task.request.id, str(e))) raise self.retry(exc=e, countdown=60)
def checksum_file(self, file_path): try: with open(file_path) as f: digest = hashlib.sha1() while True: bytes = f.read(8 * 1024) if bytes == "": break digest.update(bytes) return digest.hexdigest() except Exception, e: logger.error('checksum_file(file_path=%s) unexpectedly failed (task id ' '%s): %s, retrying in 60 s...' % (file_path, checksum_file.request.id, str(e))) raise self.retry(exc=e, countdown=60)
def mark_request_deletable(self, r_uuid): try: r = tasks.session.query(DownloadRequest).get(r_uuid) logger.info('marking %s request %s as deletable' % (r.state, r_uuid)) r.is_deletable = True tasks.session.commit() # since there might be pending requests that wait for space to # be freed up: logger.debug('=> invoking scheduler') schedule_tasks.delay() except Exception, e: logger.error('mark_request_deletable(uuid=%s) unexpectedly failed ' '(task id %s): %s, retrying in 60 s...' % (r_uuid, mark_request_deletable.request.id, str(e))) raise self.retry(exc=e, countdown=60)
def estimate_size(self, file_name): try: logger.info('calling MARS to estimate size for %s' % file_name) size = None re_size = re.compile(r'^size=([\d]+);$') mars_request = create_mars_request(verb='LIST', file_name=file_name) mars_request.params['OUTPUT'] = 'COST' logger.debug('mars_request: %s' % mars_request) log_fn = logger.debug for rc,fd,l in util.exec_proc([ 'mars' ], logger, stdin=str(mars_request)): if fd == 2: log_fn = logger.warn if l is not None: log_fn(l.strip()) if size is None and fd == 1 and l is not None: m = re_size.match(l) if m: size = int(m.group(1).replace(",", "")) assert rc is not None and fd is None and l is None # don't trust size if mars returns non-zero if rc != 0 or size is None: logger.error('failed to compute size, rc = %d' % rc) logger.error('MARS request was: %s' % mars_request) raise TaskFailure('failed to compute size, rc = %d' % rc) elif size == 0: logger.error('size is %d' % size) logger.error('MARS request was: %s' % mars_request) raise HardTaskFailure('size is %d' % size) est_size = size * config.FILE_SIZE_WEIGHT + config.FILE_SIZE_EXTRA logger.debug('MARS reported size: %d bytes, after compensating: size * ' '%d + %d = %d' % \ (size, config.FILE_SIZE_WEIGHT, config.FILE_SIZE_EXTRA, est_size)) logger.info('size is %d' % est_size) return est_size except HardTaskFailure: raise except Exception, e: logger.error('estimate_size(file_name=%s) unexpectedly failed (task id ' '%s): %s, retrying in 60 s...' % (file_name, estimate_size.request.id, str(e))) raise self.retry(exc=e, countdown=60)
def register_request(self, uuid, openid, file_to_query): try: r = DownloadRequest(uuid, openid) logger.debug('created new request %s' % r) file_names = file_to_query.keys() registered_files = tasks.session.query(StagableFile).\ filter(StagableFile.name.in_(file_names)) logger.debug('registered files: %s' % registered_files.all()) new_file_names = set(file_names) - set(x.name for x in registered_files) logger.debug('new file names: %s' % new_file_names) new_files = [ StagableFile(file_name, file_to_query[file_name]) for file_name in new_file_names ] logger.debug('new files: %s' % new_files) registered_files.update( { StagableFile.request_count : StagableFile.request_count + 1 }, synchronize_session=False) r.files.extend(registered_files) r.files.extend(new_files) # This won't throw an integrityerror as long as this task is # single threaded. If not, there's always a risk that one or more # identical StagableFile may be inserted concurrently. tasks.session.add(r) tasks.session.commit() logger.info('registered request %s openid=%s, file_names=%s ' '(unregistered=%s)' % \ (r.uuid, openid, file_names, new_file_names)) submit_sizing_tasks(r.uuid) logger.debug('=> invoking scheduler') schedule_tasks.delay() except Exception, e: # Note that some errors will never complete OK, for example if # we try to register two or more requests with the same uuid # (unlikely) - these cases are not handled and will need # manual admin intervention (in this case removal of the # redundant requests) logger.error('register_request(uuid=%s, openid=%s, file_to_query=%s) ' 'unexpectedly failed (task id %s): %s, retrying in 60 s...' % (uuid, openid, file_to_query, register_request.request.id, str(e))) raise self.retry(exc=e, countdown=60)
def stage_file(self, file_name, target_dir, path_out): try: end_target = os.path.join(target_dir, file_name) tmp_target = os.path.join(target_dir, uuid.uuid4().get_hex()) logger.info('staging %s' % end_target) logger.debug('tmp_target: %s' % tmp_target) mars_request = create_mars_request(verb='RETRIEVE', file_name=file_name, target=tmp_target) logger.debug('mars_request: %s' % mars_request) with open(path_out, 'w') as f: for rc,fd,l in util.exec_proc([ 'mars' ], logger, stdin=str(mars_request)): if fd is not None and l is not None: if fd == 1: logger.debug('fd=%s, l=%s' % (fd, l.strip() if l else l)) else: logger.warning('fd=%s, l=%s' % (fd, l.strip() if l else l)) f.write(l) f.flush() f.flush() os.fdatasync(f.fileno()) f.close() if rc != 0: logger.error('failed to stage %s, rc = %d' % (file_name, rc)) logger.error('MARS request was: %s' % mars_request) logger.debug('removing temp file %s' % tmp_target) util.unlink(tmp_target) # FIXME: use try...finally raise TaskFailure('mars returned %d' % rc) logger.debug('moving temp file %s -> %s' % (tmp_target, end_target)) os.rename(tmp_target, end_target) logger.info('%s is staged online' % end_target) logger.debug('=> invoking scheduler') tasks.scheduler.join_staging_task.delay(stage_file.request.id) except Exception, e: logger.error('stage_file(file_name=%s, target_dir=%s, path_out=%s) ' 'unexpectedly failed (task id %s): %s, retrying in ' '60 s...' % (file_name, target_dir, path_out, stage_file.request.id, str(e))) raise self.retry(exc=e, countdown=60)
# of staging -> checksumming): sf.staging_task = Task(async_result.id, path_out) # IMPORTANT: commit immediately since stage_file # calls join_staging_tasks which looks up the task # in the db: tasks.session.commit() logger.info('=> staging task for %s is %s' % \ (sf.name, sf.staging_task.uuid)) consumed_space += offline_size available_space -= offline_size rs.state = 'dispatched' num_tasks_dispatched += 1 else: logger.info('request %s is not eliglible for dispatching since it ' 'can not be fast forwarded and at least one higher ' 'priority request is still waiting to be dispatched' % \ rs.uuid) num_tasks_deferred += 1 logger.info('scheduling iteration completed, %d tasks dispatched, %d ' 'tasks failed, %d tasks deferred. Consumed space %d bytes, ' 'total space available %d bytes' % \ (num_tasks_dispatched, num_tasks_failed, num_tasks_deferred, consumed_space, available_space)) tasks.session.commit() except Exception, e: logger.error('schedule_tasks() unexpectedly failed (task id %s): %s, ' 'retrying in 60 s...' % (schedule_tasks.request.id, str(e))) raise self.retry(exc=e, countdown=60)