def get_divide(a, b):
try:
r = a / b
return r
except Exception as e:
logger.error("calc error, {}".format(e))
return None
def join_staging_task(self, task_id):
try:
staging_task = get_task(task_id)
logger.debug('staging task for %s is: %s' % (task_id, staging_task))
# BUG: FIXME: why is this a list and not a file, we do use uselist=False
sf = staging_task.stagable_file[0]
logger.info('staging task %s completed, file is %s' % (task_id, sf.name))
handle_staging_result(staging_task, sf)
tasks.session.commit()
except Exception, e:
logger.error('join_staging_task(task_id=%s) unexpectedly failed (task '
'id %s): %s, retrying in 60 s...' %
(task_id, join_staging_task.request.id, str(e)))
raise self.retry(exc=e, countdown=60)
def checksum_file(self, file_path):
try:
with open(file_path) as f:
digest = hashlib.sha1()
while True:
bytes = f.read(8 * 1024)
if bytes == "":
break
digest.update(bytes)
return digest.hexdigest()
except Exception, e:
logger.error('checksum_file(file_path=%s) unexpectedly failed (task id '
'%s): %s, retrying in 60 s...' %
(file_path, checksum_file.request.id, str(e)))
raise self.retry(exc=e, countdown=60)
def mark_request_deletable(self, r_uuid):
try:
r = tasks.session.query(DownloadRequest).get(r_uuid)
logger.info('marking %s request %s as deletable' % (r.state, r_uuid))
r.is_deletable = True
tasks.session.commit()
# since there might be pending requests that wait for space to
# be freed up:
logger.debug('=> invoking scheduler')
schedule_tasks.delay()
except Exception, e:
logger.error('mark_request_deletable(uuid=%s) unexpectedly failed '
'(task id %s): %s, retrying in 60 s...' %
(r_uuid, mark_request_deletable.request.id, str(e)))
raise self.retry(exc=e, countdown=60)
def estimate_size(self, file_name):
try:
logger.info('calling MARS to estimate size for %s' % file_name)
size = None
re_size = re.compile(r'^size=([\d]+);$')
mars_request = create_mars_request(verb='LIST', file_name=file_name)
mars_request.params['OUTPUT'] = 'COST'
logger.debug('mars_request: %s' % mars_request)
log_fn = logger.debug
for rc,fd,l in util.exec_proc([ 'mars' ], logger, stdin=str(mars_request)):
if fd == 2:
log_fn = logger.warn
if l is not None:
log_fn(l.strip())
if size is None and fd == 1 and l is not None:
m = re_size.match(l)
if m:
size = int(m.group(1).replace(",", ""))
assert rc is not None and fd is None and l is None
# don't trust size if mars returns non-zero
if rc != 0 or size is None:
logger.error('failed to compute size, rc = %d' % rc)
logger.error('MARS request was: %s' % mars_request)
raise TaskFailure('failed to compute size, rc = %d' % rc)
elif size == 0:
logger.error('size is %d' % size)
logger.error('MARS request was: %s' % mars_request)
raise HardTaskFailure('size is %d' % size)
est_size = size * config.FILE_SIZE_WEIGHT + config.FILE_SIZE_EXTRA
logger.debug('MARS reported size: %d bytes, after compensating: size * '
'%d + %d = %d' % \
(size, config.FILE_SIZE_WEIGHT, config.FILE_SIZE_EXTRA,
est_size))
logger.info('size is %d' % est_size)
return est_size
except HardTaskFailure:
raise
except Exception, e:
logger.error('estimate_size(file_name=%s) unexpectedly failed (task id '
'%s): %s, retrying in 60 s...' %
(file_name, estimate_size.request.id, str(e)))
raise self.retry(exc=e, countdown=60)
def register_request(self, uuid, openid, file_to_query):
try:
r = DownloadRequest(uuid, openid)
logger.debug('created new request %s' % r)
file_names = file_to_query.keys()
registered_files = tasks.session.query(StagableFile).\
filter(StagableFile.name.in_(file_names))
logger.debug('registered files: %s' % registered_files.all())
new_file_names = set(file_names) - set(x.name for x in registered_files)
logger.debug('new file names: %s' % new_file_names)
new_files = [ StagableFile(file_name, file_to_query[file_name]) for file_name in new_file_names ]
logger.debug('new files: %s' % new_files)
registered_files.update(
{ StagableFile.request_count : StagableFile.request_count + 1 },
synchronize_session=False)
r.files.extend(registered_files)
r.files.extend(new_files)
# This won't throw an integrityerror as long as this task is
# single threaded. If not, there's always a risk that one or more
# identical StagableFile may be inserted concurrently.
tasks.session.add(r)
tasks.session.commit()
logger.info('registered request %s openid=%s, file_names=%s '
'(unregistered=%s)' % \
(r.uuid, openid, file_names, new_file_names))
submit_sizing_tasks(r.uuid)
logger.debug('=> invoking scheduler')
schedule_tasks.delay()
except Exception, e:
# Note that some errors will never complete OK, for example if
# we try to register two or more requests with the same uuid
# (unlikely) - these cases are not handled and will need
# manual admin intervention (in this case removal of the
# redundant requests)
logger.error('register_request(uuid=%s, openid=%s, file_to_query=%s) '
'unexpectedly failed (task id %s): %s, retrying in 60 s...' %
(uuid, openid, file_to_query, register_request.request.id,
str(e)))
raise self.retry(exc=e, countdown=60)
def stage_file(self, file_name, target_dir, path_out):
try:
end_target = os.path.join(target_dir, file_name)
tmp_target = os.path.join(target_dir, uuid.uuid4().get_hex())
logger.info('staging %s' % end_target)
logger.debug('tmp_target: %s' % tmp_target)
mars_request = create_mars_request(verb='RETRIEVE',
file_name=file_name,
target=tmp_target)
logger.debug('mars_request: %s' % mars_request)
with open(path_out, 'w') as f:
for rc,fd,l in util.exec_proc([ 'mars' ], logger, stdin=str(mars_request)):
if fd is not None and l is not None:
if fd == 1:
logger.debug('fd=%s, l=%s' % (fd, l.strip() if l else l))
else:
logger.warning('fd=%s, l=%s' % (fd, l.strip() if l else l))
f.write(l)
f.flush()
f.flush()
os.fdatasync(f.fileno())
f.close()
if rc != 0:
logger.error('failed to stage %s, rc = %d' % (file_name, rc))
logger.error('MARS request was: %s' % mars_request)
logger.debug('removing temp file %s' % tmp_target)
util.unlink(tmp_target) # FIXME: use try...finally
raise TaskFailure('mars returned %d' % rc)
logger.debug('moving temp file %s -> %s' % (tmp_target, end_target))
os.rename(tmp_target, end_target)
logger.info('%s is staged online' % end_target)
logger.debug('=> invoking scheduler')
tasks.scheduler.join_staging_task.delay(stage_file.request.id)
except Exception, e:
logger.error('stage_file(file_name=%s, target_dir=%s, path_out=%s) '
'unexpectedly failed (task id %s): %s, retrying in '
'60 s...' %
(file_name, target_dir, path_out, stage_file.request.id,
str(e)))
raise self.retry(exc=e, countdown=60)
# of staging -> checksumming):
sf.staging_task = Task(async_result.id, path_out)
# IMPORTANT: commit immediately since stage_file
# calls join_staging_tasks which looks up the task
# in the db:
tasks.session.commit()
logger.info('=> staging task for %s is %s' % \
(sf.name, sf.staging_task.uuid))
consumed_space += offline_size
available_space -= offline_size
rs.state = 'dispatched'
num_tasks_dispatched += 1
else:
logger.info('request %s is not eliglible for dispatching since it '
'can not be fast forwarded and at least one higher '
'priority request is still waiting to be dispatched' % \
rs.uuid)
num_tasks_deferred += 1
logger.info('scheduling iteration completed, %d tasks dispatched, %d '
'tasks failed, %d tasks deferred. Consumed space %d bytes, '
'total space available %d bytes' % \
(num_tasks_dispatched, num_tasks_failed, num_tasks_deferred,
consumed_space, available_space))
tasks.session.commit()
except Exception, e:
logger.error('schedule_tasks() unexpectedly failed (task id %s): %s, '
'retrying in 60 s...' %
(schedule_tasks.request.id, str(e)))
raise self.retry(exc=e, countdown=60)
