def set_blob(self, prop, data=None, size=None, **kwargs): if not exists(self._root): os.makedirs(self._root) path = join(self._root, prop + PropertyMeta.BLOB_SUFFIX) meta = PropertyMeta(**kwargs) if data is None: if exists(path): os.unlink(path) elif isinstance(data, PropertyMeta): data.update(meta) meta = data else: digest = hashlib.sha1() if hasattr(data, 'read'): if size is None: size = sys.maxint self._set_blob_by_stream(digest, data, size, path) elif isabs(data) and exists(data): self._set_blob_by_path(digest, data, path) else: with util.new_file(path) as f: f.write(data) digest.update(data) meta['digest'] = digest.hexdigest() self.set(prop, **meta)
def presolve(names): for repo in _get_repos(obs_presolve_project.value): for arch in repo['arches']: dirname = join(obs_presolve_path.value, repo['name'], arch) if not exists(dirname): os.makedirs(dirname) for package in names: try: response = _request('GET', ['resolve'], params={ 'project': obs_presolve_project.value, 'repository': repo['name'], 'arch': arch, 'package': package, 'withdeps': '1', 'exclude': 'sugar', }) except Exception: util.exception('Failed to resolve %s:%s:%s for presolving', repo['name'], arch, package) continue deps_graph = [] for pkg in response.findall('binary'): deps_graph.append(dict(pkg.items())) with util.new_file(join(dirname, package)) as f: json.dump(deps_graph, f)
def _sync(self): if os.stat(self._files_path).st_mtime <= self._stamp: return new_files = set() updates = 0 deletes = 0 # Populate list of new files at first for root, __, files in os.walk(self._files_path): coroutine.dispatch() rel_root = relpath(root, self._files_path) if rel_root == '.': rel_root = '' else: rel_root += os.sep for filename in files: coroutine.dispatch() path = join(root, filename) if os.lstat(path).st_mtime > self._stamp: new_files.add(rel_root + filename) # Check for updates for already tracked files tail = [] for pos, (__, rel_path, mtime) in enumerate(self._index[:]): coroutine.dispatch() path = join(self._files_path, rel_path) existing = lexists(path) if existing == (mtime >= 0) and \ (not existing or os.lstat(path).st_mtime == mtime): continue if existing: new_files.discard(rel_path) pos -= len(tail) self._index = self._index[:pos] + self._index[pos + 1:] tail.append([ self._seqno.next(), rel_path, int(os.lstat(path).st_mtime) if existing else -1, ]) if existing: updates += 1 else: deletes += 1 self._index.extend(tail) _logger.debug('Updated %r index: new=%r updates=%r deletes=%r', self._files_path, len(self._files_path), updates, deletes) # Finally, add new files for rel_path in sorted(new_files): coroutine.dispatch() mtime = os.lstat(join(self._files_path, rel_path)).st_mtime self._index.append([self._seqno.next(), rel_path, mtime]) self._stamp = os.stat(self._files_path).st_mtime if self._seqno.commit(): with util.new_file(self._index_path) as f: json.dump((self._index, self._stamp), f)
def commit(self): dir_path = dirname(self._path) if dir_path and not exists(dir_path): os.makedirs(dir_path) with util.new_file(self._path) as f: json.dump(self, f) f.flush() os.fsync(f.fileno())
def _set_blob_by_stream(self, digest, stream, size, path): with util.new_file(path) as f: while size > 0: chunk = stream.read(min(size, BUFFER_SIZE)) if not chunk: break f.write(chunk) size -= len(chunk) if digest is not None: digest.update(chunk)
def set(self, prop, mtime=None, path=None, content=None, **meta): if not exists(self._root): os.makedirs(self._root) meta_path = join(self._root, prop) blob_path = join(self._root, prop + PropertyMeta.BLOB_SUFFIX) if content is not None: with util.new_file(blob_path) as f: f.write(b64decode(content)) elif path and exists(path): util.cptree(path, blob_path) with util.new_file(meta_path) as f: pickle.dump(meta, f) if mtime: os.utime(meta_path, (mtime, mtime)) if prop == 'guid': if not mtime: mtime = time.time() # Touch directory to let it possible to crawl it on startup # when index was not previously closed properly os.utime(join(self._root, '..'), (mtime, mtime))
def commit(self): """Store current seqno value in a file. :returns: `True` if commit was happened """ if self._value == self._orig_value: return False with util.new_file(self._path) as f: f.write(str(self._value)) f.flush() os.fsync(f.fileno()) self._orig_value = self._value return True
def push(self, record): cmd = record.get('cmd') if cmd == 'files_push': blob = record['blob'] path = join(self._files_path, record['path']) if not exists(dirname(path)): os.makedirs(dirname(path)) with util.new_file(path) as f: while True: chunk = blob.read(BUFFER_SIZE) if not chunk: break f.write(chunk) elif cmd == 'files_delete': path = join(self._files_path, record['path']) if exists(path): os.unlink(path) elif cmd == 'files_commit': self.sequence.exclude(record['sequence']) self.sequence.commit()
def _save_layout(self): path = join(self._root, 'layout') with util.new_file(path) as f: f.write(str(_LAYOUT_VERSION))