def download(self, file_id): ''' Retrieve file stored in the service @param file_id: id of the file that will be downloaded @return: file ''' identifier = file_id.get_identifier() identifier = identifier.split('/') if len(identifier) != 4: raise ValueError("Unknown file to be accessed") package_name = os.path.basename(identifier[0]) branch = os.path.basename(identifier[1]) commit = self._ident2commit(os.path.basename(identifier[2])) f = os.path.basename(identifier[3]) if not f.endswith('.spec') and not f.endswith('.patch'): raise ValueError("Unknown file to be accessed") path = os.path.join(self.pkg_dir, package_name) file_path = os.path.join(path, f) log.debug("downloading '%s'" % (file_path, )) with self.get_lock(package_name): self._git_tree_prepare(package_name, branch, commit) with open(file_path, 'rb') as f: content = f.read() return content
def main(self): with self.get_system() as system: golang_pkgs = system.async_call.goland_package_listing() stored_projects = system.async_call.api_project_listing() for pkg in golang_pkgs.result: if not pkg['name'].startswith('golang-github-'): log.warning("Skipping %s" % pkg['name']) # TODO: remove once support for mercurial and full package->upstream translation will be available continue print("Inspecting '%s'" % pkg['name']) upstream_url = system.async_call.golang_package2upstream(pkg['name']) if pkg['name'] in stored_projects.result: stored_commits = system.async_call.api_project_commit_listing(pkg['name']) else: stored_commits = None scm_log = system.async_call.scm_log(upstream_url.result) for commit in tqdm(scm_log.result): log.debug("Commit %s project %s" % (commit['hash'], pkg['name'])) if not stored_commits or commit not in stored_commits.result: file_id = system.async_call.scm_store(upstream_url.result, commit['hash']) api = system.async_call.api_analysis(file_id.result) system.async_call.api_store_project(pkg['name'], commit['hash'], commit['time'], api.result, api.meta)
def download(self, file_id): """ Retrieve file stored in the service @param file_id: id of the file that will be downloaded @return: file """ identifier = file_id.get_identifier() identifier = identifier.split("/") if len(identifier) != 4: raise ValueError("Unknown file to be accessed") package_name = os.path.basename(identifier[0]) branch = os.path.basename(identifier[1]) commit = self._ident2commit(os.path.basename(identifier[2])) f = os.path.basename(identifier[3]) if not f.endswith(".spec") and not f.endswith(".patch"): raise ValueError("Unknown file to be accessed") path = os.path.join(self.pkg_dir, package_name) file_path = os.path.join(path, f) log.debug("downloading '%s'" % (file_path,)) with self.get_lock(package_name): self._git_tree_prepare(package_name, branch, commit) with open(file_path, "rb") as f: content = f.read() return content
def destruct(self): if self._connection is not None: log.debug("closing connection to '%s'" % self._service_name) self._connection.__del__() if self._instance is not None: log.debug("destruction of a local instance of service '%s'" % self._service_name) self._instance.__del__()
def signal_startup(cls, config): cls.scm_dir = config.get('scm-dir', DEFAULT_SCM_DIR) if os.path.isdir(cls.scm_dir): shutil.rmtree(cls.scm_dir) os.mkdir(cls.scm_dir) cls.scm_dir_size = config.get('scm-dir-size', DEFAULT_SCM_DIR_SIZE) cls.dircache = Dircache(cls.scm_dir, cls.scm_dir_size) log.debug("Dircache size %sB " % cls.dircache.get_max_size()) log.debug("Dircache path '%s'" % cls.dircache.get_path())
def _download_tarball(self, tarball_url, filename): log.debug("Downloading from %s" % (tarball_url,)) response = urllib2.urlopen(tarball_url) blob = response.read() h = blob_hash(blob) self.dircache.store(blob, filename) return FileId.construct(self, self.dircache.get_file_path(filename), hash_ = h)
def __getattr__(self, name): if name == 'async_call': log.debug("Preparing asynchronous call") return ConnectionCallAsync(self) elif name == 'call': log.debug("Preparing synchronous call") return ConnectionCallSync(self) else: return getattr(System, name)
def download(self, file_id, path): # TODO: this can be extended with IP/port check once multiple services of a same type will be available log.debug("Downloading file '%s' to '%s'", file_id.get_raw(), path) conn = self.get_connection(file_id.get_service_name()) call = conn.get_action('download', async=False) blob = call(file_id.get_raw()) with open(path, 'wb') as f: f.write(blob) return File.get_representation(path, file_id)
def __init__(self, config, system_json_file, service=False): self._config = config self._connections = {} self._service = service with open(system_json_file, 'r') as f: self._system = json.load(f) log.debug("Config file:\n%s\n" % dict2json(config)) log.debug("System:\n%s\n" % dict2json(self._system))
def main(self): with self.get_system() as system: golang_pkgs = system.async_call.goland_package_listing() stored_projects = system.async_call.deps_project_listing() for pkg in golang_pkgs.result: if not pkg['name'].startswith('golang-github-'): log.warning("Skipping %s" % pkg['name']) # TODO: remove once support for mercurial and full package->upstream translation will be available continue try: raise ValueError("value error") print("Inspecting '%s'" % pkg['name']) upstream_url = system.async_call.golang_package2upstream( pkg['name']) if pkg['name'] in stored_projects.result: stored_commits = system.async_call.deps_project_commit_listing( pkg['name']) else: stored_commits = None scm_log = system.async_call.scm_log( upstream_url.result, max_depth=self.max_depth, since_date=self.since_date) for commit in tqdm(scm_log.result): log.debug("Commit %s project %s" % (commit['hash'], pkg['name'])) if not stored_commits or commit not in stored_commits.result: file_id = system.async_call.scm_store( upstream_url.result, commit['hash']) deps = system.async_call.deps_analysis( file_id.result) system.async_call.deps_store_project( pkg['name'], commit['hash'], commit['time'], deps.result, deps.meta) except: exc_info = sys.exc_info() if self.skip_errors: log.error(exc_info[2].print_exc()) else: raise exc_info
def _common_get(self, url, filename): ret = ServiceResult() with self.get_lock(filename): if self.dircache.is_available(filename): ret.result = FileId.construct(self, self.dircache.get_file_path(filename)) elif remote_exists(url): log.debug("Downloading from %s" % (url,)) response = urllib2.urlopen(url) blob = response.read() h = blob_hash(blob) self.dircache.store(blob, filename) ret.result = FileId.construct(self, self.dircache.get_file_path(filename), hash_ = h) else: raise KeyError("Desired file '%s' does not exist ( %s )" % (filename, url)) ret.meta['origin'] = url return ret
def main(self): with self.get_system() as system: golang_pkgs = system.async_call.goland_package_listing() stored_projects = system.async_call.deps_project_listing() for pkg in golang_pkgs.result: if not pkg['name'].startswith('golang-github-'): log.warning("Skipping %s" % pkg['name']) # TODO: remove once support for mercurial and full package->upstream translation will be available continue try: raise ValueError("value error") print("Inspecting '%s'" % pkg['name']) upstream_url = system.async_call.golang_package2upstream(pkg['name']) if pkg['name'] in stored_projects.result: stored_commits = system.async_call.deps_project_commit_listing(pkg['name']) else: stored_commits = None scm_log = system.async_call.scm_log(upstream_url.result, max_depth=self.max_depth, since_date=self.since_date) for commit in tqdm(scm_log.result): log.debug("Commit %s project %s" % (commit['hash'], pkg['name'])) if not stored_commits or commit not in stored_commits.result: file_id = system.async_call.scm_store(upstream_url.result, commit['hash']) deps = system.async_call.deps_analysis(file_id.result) system.async_call.deps_store_project(pkg['name'], commit['hash'], commit['time'], deps.result, deps.meta) except: exc_info = sys.exc_info() if self.skip_errors: log.error(exc_info[2].print_exc()) else: raise exc_info
def _connect(self): log.debug("Connecting to service '%s', %s:%s", self._service_name, self.host, self.port) self._connection = rpyc.connect(self.host, self.port)
def signal_startup(cls, config): cls.tarball_dir = config.get('tarball-dir', DEFAULT_TARBALL_DIR) cls.tarball_dir_size = config.get('tarball-dir-size', DEFAULT_TARBALL_DIR_SIZE) cls.dircache = Dircache(cls.tarball_dir, cls.tarball_dir_size) log.debug("Dircache size %sB " % cls.dircache.get_max_size()) log.debug("Dircache path '%s'" % cls.dircache.get_path())