def get_file_chunk(data_type, file_name, offset, size): """Return a chunk of given file. """ logger.info('... get_file_chunk(%s, %s, %d, %d)'%(data_type, file_name, offset, size)) path = datex_config.get_path(data_type)[0] fob = open(os.path.join(path, file_name)) fob.seek(offset) buf = fob.read(size) fob.close() return rpc.Binary(buf)
def younger_than_stamp_files(): """Uses glob polling to get new files. """ fdir, fglob = datex_config.get_path(datatype) del fglob fstamp = stamp_config.get_last_stamp() for fname, ftime in _get_file_list(datatype, time_start=fstamp + TIME_EPSILON): if datex_config.distribute(datatype): yield os.path.join(fdir, fname) stamp_config.update_last_stamp(ftime)
def get_file_chunk(data_type, file_name, offset, size): """Return a chunk of given file. """ logger.info('... get_file_chunk(%s, %s, %d, %d)' % (data_type, file_name, offset, size)) path = datex_config.get_path(data_type)[0] fob = open(os.path.join(path, file_name)) fob.seek(offset) buf = fob.read(size) fob.close() return rpc.Binary(buf)
def get_file_md5(data_type, file_name): """Calculate md5 of a given file and return it. """ logger.info('... get_file_md5(%s, %s)', data_type, file_name) path = datex_config.get_path(data_type)[0] fob = open(os.path.join(path, file_name)) md5 = hashlib.md5() while True: buf = fob.read(128) if not buf: break md5.update(buf) fob.close() return md5.hexdigest()
def _get_file_list(data_type, time_start=None, time_end=None): """Return a list of files for give datatype and time interval (internal). """ if time_start is None: time_start = datetime.utcfromtimestamp(0) if time_end is None: time_end = datetime.utcnow() path, file_glob = datex_config.get_path(data_type) min_age = datex_config.get_min_age(data_type) max_age = datex_config.get_max_age(data_type) file_list = glob(os.path.join(path, file_glob)) result = [] for fname in file_list: if not os.path.isfile(fname): continue ftime = datetime.utcfromtimestamp(os.stat(fname).st_mtime) if((time_start < ftime < time_end) and (min_age < (datetime.utcnow() - ftime).seconds < max_age)): result.append((fname, ftime)) result = sorted(result, lambda x, y: cmp(x[1], y[1])) return result
def _get_file_list(data_type, time_start=None, time_end=None): """Return a list of files for give datatype and time interval (internal). """ if time_start is None: time_start = datetime.utcfromtimestamp(0) if time_end is None: time_end = datetime.utcnow() path, file_glob = datex_config.get_path(data_type) min_age = datex_config.get_min_age(data_type) max_age = datex_config.get_max_age(data_type) file_list = glob(os.path.join(path, file_glob)) result = [] for fname in file_list: if not os.path.isfile(fname): continue ftime = datetime.utcfromtimestamp(os.stat(fname).st_mtime) if ((time_start < ftime < time_end) and (min_age < (datetime.utcnow() - ftime).seconds < max_age)): result.append((fname, ftime)) result = sorted(result, lambda x, y: cmp(x[1], y[1])) return result