def get_file_chunk(data_type, file_name, offset, size):
"""Return a chunk of given file.
"""
logger.info('... get_file_chunk(%s, %s, %d, %d)'%(data_type, file_name,
offset, size))
path = datex_config.get_path(data_type)[0]
fob = open(os.path.join(path, file_name))
fob.seek(offset)
buf = fob.read(size)
fob.close()
return rpc.Binary(buf)
def younger_than_stamp_files():
"""Uses glob polling to get new files.
"""
fdir, fglob = datex_config.get_path(datatype)
del fglob
fstamp = stamp_config.get_last_stamp()
for fname, ftime in _get_file_list(datatype,
time_start=fstamp + TIME_EPSILON):
if datex_config.distribute(datatype):
yield os.path.join(fdir, fname)
stamp_config.update_last_stamp(ftime)
def younger_than_stamp_files():
"""Uses glob polling to get new files.
"""
fdir, fglob = datex_config.get_path(datatype)
del fglob
fstamp = stamp_config.get_last_stamp()
for fname, ftime in _get_file_list(datatype,
time_start=fstamp + TIME_EPSILON):
if datex_config.distribute(datatype):
yield os.path.join(fdir, fname)
stamp_config.update_last_stamp(ftime)
def get_file_chunk(data_type, file_name, offset, size):
"""Return a chunk of given file.
"""
logger.info('... get_file_chunk(%s, %s, %d, %d)' %
(data_type, file_name, offset, size))
path = datex_config.get_path(data_type)[0]
fob = open(os.path.join(path, file_name))
fob.seek(offset)
buf = fob.read(size)
fob.close()
return rpc.Binary(buf)
def get_file_md5(data_type, file_name):
"""Calculate md5 of a given file and return it.
"""
logger.info('... get_file_md5(%s, %s)', data_type, file_name)
path = datex_config.get_path(data_type)[0]
fob = open(os.path.join(path, file_name))
md5 = hashlib.md5()
while True:
buf = fob.read(128)
if not buf:
break
md5.update(buf)
fob.close()
return md5.hexdigest()
def get_file_md5(data_type, file_name):
"""Calculate md5 of a given file and return it.
"""
logger.info('... get_file_md5(%s, %s)', data_type, file_name)
path = datex_config.get_path(data_type)[0]
fob = open(os.path.join(path, file_name))
md5 = hashlib.md5()
while True:
buf = fob.read(128)
if not buf:
break
md5.update(buf)
fob.close()
return md5.hexdigest()
def _get_file_list(data_type, time_start=None, time_end=None):
"""Return a list of files for give datatype and time interval (internal).
"""
if time_start is None:
time_start = datetime.utcfromtimestamp(0)
if time_end is None:
time_end = datetime.utcnow()
path, file_glob = datex_config.get_path(data_type)
min_age = datex_config.get_min_age(data_type)
max_age = datex_config.get_max_age(data_type)
file_list = glob(os.path.join(path, file_glob))
result = []
for fname in file_list:
if not os.path.isfile(fname):
continue
ftime = datetime.utcfromtimestamp(os.stat(fname).st_mtime)
if((time_start < ftime < time_end) and
(min_age < (datetime.utcnow() - ftime).seconds < max_age)):
result.append((fname, ftime))
result = sorted(result, lambda x, y: cmp(x[1], y[1]))
return result
def _get_file_list(data_type, time_start=None, time_end=None):
"""Return a list of files for give datatype and time interval (internal).
"""
if time_start is None:
time_start = datetime.utcfromtimestamp(0)
if time_end is None:
time_end = datetime.utcnow()
path, file_glob = datex_config.get_path(data_type)
min_age = datex_config.get_min_age(data_type)
max_age = datex_config.get_max_age(data_type)
file_list = glob(os.path.join(path, file_glob))
result = []
for fname in file_list:
if not os.path.isfile(fname):
continue
ftime = datetime.utcfromtimestamp(os.stat(fname).st_mtime)
if ((time_start < ftime < time_end)
and (min_age < (datetime.utcnow() - ftime).seconds < max_age)):
result.append((fname, ftime))
result = sorted(result, lambda x, y: cmp(x[1], y[1]))
return result
