def download(self, file_id):
'''
Retrieve file stored in the service
@param file_id: id of the file that will be downloaded
@return: file
'''
identifier = file_id.get_identifier()
identifier = identifier.split('/')
if len(identifier) != 4:
raise ValueError("Unknown file to be accessed")
package_name = os.path.basename(identifier[0])
branch = os.path.basename(identifier[1])
commit = self._ident2commit(os.path.basename(identifier[2]))
f = os.path.basename(identifier[3])
if not f.endswith('.spec') and not f.endswith('.patch'):
raise ValueError("Unknown file to be accessed")
path = os.path.join(self.pkg_dir, package_name)
file_path = os.path.join(path, f)
log.debug("downloading '%s'" % (file_path, ))
with self.get_lock(package_name):
self._git_tree_prepare(package_name, branch, commit)
with open(file_path, 'rb') as f:
content = f.read()
return content
def main(self):
with self.get_system() as system:
golang_pkgs = system.async_call.goland_package_listing()
stored_projects = system.async_call.api_project_listing()
for pkg in golang_pkgs.result:
if not pkg['name'].startswith('golang-github-'):
log.warning("Skipping %s" % pkg['name'])
# TODO: remove once support for mercurial and full package->upstream translation will be available
continue
print("Inspecting '%s'" % pkg['name'])
upstream_url = system.async_call.golang_package2upstream(pkg['name'])
if pkg['name'] in stored_projects.result:
stored_commits = system.async_call.api_project_commit_listing(pkg['name'])
else:
stored_commits = None
scm_log = system.async_call.scm_log(upstream_url.result)
for commit in tqdm(scm_log.result):
log.debug("Commit %s project %s" % (commit['hash'], pkg['name']))
if not stored_commits or commit not in stored_commits.result:
file_id = system.async_call.scm_store(upstream_url.result, commit['hash'])
api = system.async_call.api_analysis(file_id.result)
system.async_call.api_store_project(pkg['name'], commit['hash'], commit['time'],
api.result, api.meta)
def download(self, file_id):
"""
Retrieve file stored in the service
@param file_id: id of the file that will be downloaded
@return: file
"""
identifier = file_id.get_identifier()
identifier = identifier.split("/")
if len(identifier) != 4:
raise ValueError("Unknown file to be accessed")
package_name = os.path.basename(identifier[0])
branch = os.path.basename(identifier[1])
commit = self._ident2commit(os.path.basename(identifier[2]))
f = os.path.basename(identifier[3])
if not f.endswith(".spec") and not f.endswith(".patch"):
raise ValueError("Unknown file to be accessed")
path = os.path.join(self.pkg_dir, package_name)
file_path = os.path.join(path, f)
log.debug("downloading '%s'" % (file_path,))
with self.get_lock(package_name):
self._git_tree_prepare(package_name, branch, commit)
with open(file_path, "rb") as f:
content = f.read()
return content
def destruct(self):
if self._connection is not None:
log.debug("closing connection to '%s'" % self._service_name)
self._connection.__del__()
if self._instance is not None:
log.debug("destruction of a local instance of service '%s'" % self._service_name)
self._instance.__del__()
def destruct(self):
if self._connection is not None:
log.debug("closing connection to '%s'" % self._service_name)
self._connection.__del__()
if self._instance is not None:
log.debug("destruction of a local instance of service '%s'" %
self._service_name)
self._instance.__del__()
def signal_startup(cls, config):
cls.scm_dir = config.get('scm-dir', DEFAULT_SCM_DIR)
if os.path.isdir(cls.scm_dir):
shutil.rmtree(cls.scm_dir)
os.mkdir(cls.scm_dir)
cls.scm_dir_size = config.get('scm-dir-size', DEFAULT_SCM_DIR_SIZE)
cls.dircache = Dircache(cls.scm_dir, cls.scm_dir_size)
log.debug("Dircache size %sB " % cls.dircache.get_max_size())
log.debug("Dircache path '%s'" % cls.dircache.get_path())
def _download_tarball(self, tarball_url, filename):
log.debug("Downloading from %s" % (tarball_url,))
response = urllib2.urlopen(tarball_url)
blob = response.read()
h = blob_hash(blob)
self.dircache.store(blob, filename)
return FileId.construct(self, self.dircache.get_file_path(filename), hash_ = h)
def __getattr__(self, name):
if name == 'async_call':
log.debug("Preparing asynchronous call")
return ConnectionCallAsync(self)
elif name == 'call':
log.debug("Preparing synchronous call")
return ConnectionCallSync(self)
else:
return getattr(System, name)
def download(self, file_id, path):
# TODO: this can be extended with IP/port check once multiple services of a same type will be available
log.debug("Downloading file '%s' to '%s'", file_id.get_raw(), path)
conn = self.get_connection(file_id.get_service_name())
call = conn.get_action('download', async=False)
blob = call(file_id.get_raw())
with open(path, 'wb') as f:
f.write(blob)
return File.get_representation(path, file_id)
def __init__(self, config, system_json_file, service=False):
self._config = config
self._connections = {}
self._service = service
with open(system_json_file, 'r') as f:
self._system = json.load(f)
log.debug("Config file:\n%s\n" % dict2json(config))
log.debug("System:\n%s\n" % dict2json(self._system))
def main(self):
with self.get_system() as system:
golang_pkgs = system.async_call.goland_package_listing()
stored_projects = system.async_call.deps_project_listing()
for pkg in golang_pkgs.result:
if not pkg['name'].startswith('golang-github-'):
log.warning("Skipping %s" % pkg['name'])
# TODO: remove once support for mercurial and full package->upstream translation will be available
continue
try:
raise ValueError("value error")
print("Inspecting '%s'" % pkg['name'])
upstream_url = system.async_call.golang_package2upstream(
pkg['name'])
if pkg['name'] in stored_projects.result:
stored_commits = system.async_call.deps_project_commit_listing(
pkg['name'])
else:
stored_commits = None
scm_log = system.async_call.scm_log(
upstream_url.result,
max_depth=self.max_depth,
since_date=self.since_date)
for commit in tqdm(scm_log.result):
log.debug("Commit %s project %s" %
(commit['hash'], pkg['name']))
if not stored_commits or commit not in stored_commits.result:
file_id = system.async_call.scm_store(
upstream_url.result, commit['hash'])
deps = system.async_call.deps_analysis(
file_id.result)
system.async_call.deps_store_project(
pkg['name'], commit['hash'], commit['time'],
deps.result, deps.meta)
except:
exc_info = sys.exc_info()
if self.skip_errors:
log.error(exc_info[2].print_exc())
else:
raise exc_info
def _common_get(self, url, filename):
ret = ServiceResult()
with self.get_lock(filename):
if self.dircache.is_available(filename):
ret.result = FileId.construct(self, self.dircache.get_file_path(filename))
elif remote_exists(url):
log.debug("Downloading from %s" % (url,))
response = urllib2.urlopen(url)
blob = response.read()
h = blob_hash(blob)
self.dircache.store(blob, filename)
ret.result = FileId.construct(self, self.dircache.get_file_path(filename), hash_ = h)
else:
raise KeyError("Desired file '%s' does not exist ( %s )" % (filename, url))
ret.meta['origin'] = url
return ret
def main(self):
with self.get_system() as system:
golang_pkgs = system.async_call.goland_package_listing()
stored_projects = system.async_call.deps_project_listing()
for pkg in golang_pkgs.result:
if not pkg['name'].startswith('golang-github-'):
log.warning("Skipping %s" % pkg['name'])
# TODO: remove once support for mercurial and full package->upstream translation will be available
continue
try:
raise ValueError("value error")
print("Inspecting '%s'" % pkg['name'])
upstream_url = system.async_call.golang_package2upstream(pkg['name'])
if pkg['name'] in stored_projects.result:
stored_commits = system.async_call.deps_project_commit_listing(pkg['name'])
else:
stored_commits = None
scm_log = system.async_call.scm_log(upstream_url.result,
max_depth=self.max_depth,
since_date=self.since_date)
for commit in tqdm(scm_log.result):
log.debug("Commit %s project %s" % (commit['hash'], pkg['name']))
if not stored_commits or commit not in stored_commits.result:
file_id = system.async_call.scm_store(upstream_url.result, commit['hash'])
deps = system.async_call.deps_analysis(file_id.result)
system.async_call.deps_store_project(pkg['name'], commit['hash'], commit['time'],
deps.result, deps.meta)
except:
exc_info = sys.exc_info()
if self.skip_errors:
log.error(exc_info[2].print_exc())
else:
raise exc_info
def _connect(self):
log.debug("Connecting to service '%s', %s:%s", self._service_name,
self.host, self.port)
self._connection = rpyc.connect(self.host, self.port)
def _connect(self):
log.debug("Connecting to service '%s', %s:%s", self._service_name, self.host, self.port)
self._connection = rpyc.connect(self.host, self.port)
def signal_startup(cls, config):
cls.tarball_dir = config.get('tarball-dir', DEFAULT_TARBALL_DIR)
cls.tarball_dir_size = config.get('tarball-dir-size', DEFAULT_TARBALL_DIR_SIZE)
cls.dircache = Dircache(cls.tarball_dir, cls.tarball_dir_size)
log.debug("Dircache size %sB " % cls.dircache.get_max_size())
log.debug("Dircache path '%s'" % cls.dircache.get_path())
