def install_virtualenv_contents_setup(venv_path, package_repository_path, package_list_path): """Install or upgrade requirements into an existing virtualenv using setup.py""" # Create temporary working directory work_dir = sudo('mktemp -d') # Read contents of file with package names reqs_file_contents = run('cat "{package_list_path}"'.format(**locals())) for fn in reqs_file_contents.split('\n'): fn = fn.strip() if fn == '': continue # Unpack the package into the temporary directory with cd(work_dir): sudo(get_extract_command(posixjoin(package_repository_path, fn))) # Run setup.py from extracted package package_name = remove_package_extension(fn) extracted_package_dir = posixjoin(work_dir, package_name) with cd(extracted_package_dir): log_name = posixjoin(work_dir, package_name + '.log') sudo('{venv_path}/bin/python setup.py install > {log_name} 2>&1'. format(**locals()))
def find_tarball(self, modulename, max_version, wantchecksum): if modulename not in self.modules: return None, None, None resp = perform_request( posixjoin(self.baseurl, modulename, 'cache.json')) versions = resp.json()[1][modulename] latest = get_latest_version(versions.keys(), max_version) if not latest: return None, None, None extensions = ['tar.xz', 'tar.bz2', 'tar.gz'] for ext in extensions: if ext in versions[latest]: tarball = versions[latest][ext] break else: # unknown extension return None, None, None checksum = None if wantchecksum and 'sha256sum' in versions[latest]: resp = perform_request( posixjoin(self.baseurl, modulename, versions[latest]['sha256sum'])) basename = os.path.basename(tarball) for l in resp.text.splitlines(): l = l.split() if basename == l[1]: checksum = l[0] break return posixjoin(self.baseurl, modulename, tarball), latest, checksum
def find_tarball(self, modulename, max_version, wantchecksum): good_dir = re.compile('^([0-9]+\.)*[0-9]+/?$') def hasdirs(x): return good_dir.search(x) def fixdirs(x): return re.sub(r'^([0-9]+\.[0-9]+)/?$', r'\1', x) location = self.baseurl.format(module=modulename) while True: req = perform_request(location) files = get_links(req.text) # Check to see if we need to descend to a subdirectory newdirs = [fixdirs(dir) for dir in files if hasdirs(dir)] if newdirs: assert max_version is None or len( max_version.split('.') ) <= 2, "limit can't have micro version when the project uses subdirs" newdir = get_latest_version(newdirs, max_version) location = posixjoin(req.url, newdir, "") else: break basenames = set() tarballs = [] extensions = ['.tar.xz', '.tar.bz2', '.tar.gz'] # Has to be checked by extension first; we prefer .tar.xz over .tar.bz2 and .tar.gz for ext in extensions: for file in files: basename = file[:-len( ext)] # only valid when file ends with ext if file.endswith(ext) and basename not in basenames: basenames.add(basename) tarballs.append(file) re_tarball = r'^' + re.escape( modulename) + '[_-](([0-9]+[\.\-])*[0-9]+)(\.orig)?\.tar.*$' tarballs = [t for t in tarballs if re.search(re_tarball, t)] versions = [re.sub(re_tarball, r'\1', t) for t in tarballs] if not versions: return None, None, None version = get_latest_version(versions, max_version) index = versions.index(version) location = posixjoin(location, tarballs[index]) return location, version, None
def upimg_save(**kwargs): res = dict(code=1) try: filename = kwargs["filename"] stream = kwargs["stream"] upload_path = kwargs.get("upload_path") or "" local_basedir = get_basedir() if not filename or not stream or not local_basedir: return ValueError except (KeyError, ValueError): res.update(msg="Parameter error") else: if isinstance(upload_path, string_types): if upload_path.startswith("/"): upload_path = upload_path.lstrip('/') saveto = join(local_basedir, upload_path) if not exists(saveto): makedirs(saveto) filepath = join(saveto, filename) with open(filepath, "wb") as fp: fp.write(stream) res.update(code=0, src=url_for("static", filename=posixjoin( current_app.config['UPLOAD_FOLDER'], upload_path, filename), _external=True)) else: res.update(msg="The upload_path type error") return res
def __init__(self, preprocessors=None, postprocessors=None, settings=None): if settings.has_key('api_version'): self._api_version = 'v' + str(settings['api_version']) else: self._api_version = 'v1' if settings.has_key('instance_url'): self._base_url = settings['instance_url'] else: self._base_url = 'https://search.mapzen.com' self._default_endpoint = urljoin(self._base_url, posixjoin(self._api_version, 'search')) self._key_endpoint = urljoin(self._base_url, posixjoin(self._api_version, 'place')) self._endpoint = self._default_endpoint preprocessors = Mapzen.DEFAULT_PREPROCESSORS if preprocessors is None else preprocessors postprocessors = Mapzen.DEFAULT_POSTPROCESSORS if postprocessors is None else postprocessors GeocodeService.__init__(self, preprocessors, postprocessors, settings)
def __init__(self, preprocessors=None, postprocessors=None, settings=None): if settings.has_key('api_version'): self._api_version = 'v' + str(settings['api_version']) else: self._api_version = 'v1' if settings.has_key('instance_url'): self._base_url = settings['instance_url'] else: self._base_url = 'https://search.mapzen.com' self._default_endpoint = urljoin( self._base_url, posixjoin(self._api_version, 'search')) self._key_endpoint = urljoin(self._base_url, posixjoin(self._api_version, 'place')) self._endpoint = self._default_endpoint preprocessors = Mapzen.DEFAULT_PREPROCESSORS if preprocessors is None else preprocessors postprocessors = Mapzen.DEFAULT_POSTPROCESSORS if postprocessors is None else postprocessors GeocodeService.__init__(self, preprocessors, postprocessors, settings)
def __init__(self, preprocessors=None, postprocessors=None, settings=None): if 'api_version' in settings: self._api_version = 'v' + str(settings['api_version']) else: self._api_version = 'v1' if 'instance_url' in settings: self._base_url = settings['instance_url'] else: self._base_url = 'https://api.geocode.earth/' self._default_endpoint = urljoin( self._base_url, posixjoin(self._api_version, 'search')) self._key_endpoint = urljoin(self._base_url, posixjoin(self._api_version, 'place')) self._endpoint = self._default_endpoint preprocessors = Pelias.DEFAULT_PREPROCESSORS if preprocessors is None else preprocessors postprocessors = Pelias.DEFAULT_POSTPROCESSORS if postprocessors is None else postprocessors GeocodeService.__init__(self, preprocessors, postprocessors, settings)
def links(self, page, base="", favicon="", styles=(), **kwargs): """ Iterator[str]: <link> tags in page <head>. """ home, urlpath = self.home, self.urlpath page = page.with_suffix(".html") link = '<link rel="{}" href="{}">'.format if base: yield link("canonical", posixjoin(base, urlpath(home, page))) if favicon: yield link("icon", urlpath(page, favicon)) for sheet in styles: yield link("stylesheet", urlpath(page, sheet))
def guess_metadata_type(resource): parsed = urlparse.urlparse(resource) if not parsed.scheme: if os.path.exists(os.path.join(resource, 'repodata', 'repomd.xml')): return 'rpm-md' elif os.path.exists(os.path.join(resource, 'content')): return 'yast2' else: try: url = posixjoin(resource, 'repodata', 'repomd.xml') urllib2.urlopen(HeadRequest(url)) return 'rpm-md' except urllib2.HTTPError, e: if e.code != 404: raise e try: url = posixjoin(resource, 'content') urllib2.urlopen(HeadRequest(url)) return 'yast2' except urllib2.HTTPError, e: if e.code != 404: raise e
def urlpath(cls, page, src): """ str: URL from page to (local file or remote URL). Inputs must be absolute path-like objects. """ page, src = Path(page), posixjoin(src) if urlsplit(src).scheme: return src elif not src.startswith("/"): raise ValueError(f"ambiguous src: {src}") elif not page.is_absolute(): raise ValueError(f"ambiguous page: {page}") else: return quote(relpath(src, start=page.parent.as_posix()))
def _get_file(self, fileid, outname): """ Download an artifact into a local file """ # Create the directory to put the file, if needed (head, tail) = os.path.split(outname) if head: mkpath_recursive(head) # Download the file source_path = posixjoin(*object_remote_location(fileid)) with open(outname, 'wb') as f: self.reader.get(source_path, f) # Set permissions & modification time filedata = self.remote_index[fileid] os.chmod(outname, int(filedata[2])) os.utime(outname, (-1, float(filedata[0])))
def _update_upstream_data(self, branch, upstream_name_branches): branch_path = os.path.join(self.dest_dir, branch) if not os.path.exists(branch_path): print >> sys.stderr, 'No file %s available for requested branch %s, keeping previous data if available.' % (branch_path, branch) return (branch_id, branch_mtime) = self._get_branch_data(branch) stats = os.stat(branch_path) new_mtime = stats.st_mtime if not branch_id: # the branch does not exist, add it self.cursor.execute('''INSERT INTO branches VALUES ( NULL, ?, ? );''', (branch, new_mtime)) self.cursor.execute('''SELECT last_insert_rowid();''') branch_id = self.cursor.fetchone()[0] else: # do not update anything if the file has not changed if branch_mtime >= new_mtime: return # else update the mtime self.cursor.execute('''UPDATE branches SET mtime = ? WHERE id = ?;''', (new_mtime, branch_id)) self.cursor.execute('''SELECT * FROM upstream WHERE branch = ?;''', (branch_id,)) olddata = {} for row in self.cursor: olddata[row['name']] = (row['id'], row['version'], row['url']) # upstream data, after we've converted the names to branch names if # needed. For instance, glib:1.2.10 will translate to the "glib|1.3" # name but also to the "glib" name if it doesn't exist yet or if the # version there is lower than 1.2.10. real_upstream_data = {} re_upstream_data = re.compile('^([^:]*):([^:]+):([^:]+):(.*)$') file = open(branch_path) while True: line = file.readline() if len(line) == 0: break if self._is_line_comment(line): continue line = line[:-1] match = re_upstream_data.match(line) if not match: continue name = match.group(2) version = match.group(3) if match.group(1) == 'fallback': url = '' elif match.group(1) == 'nonfgo': url = match.group(4) elif match.group(1) == 'upstream': url = '' elif match.group(1) == 'cpan': url = posixjoin('http://cpan.perl.org/CPAN/authors/id/', match.group(4)) elif match.group(1) == 'pypi': url = match.group(4) elif match.group(1) == 'fgo': versions = version.split('.') if len(versions) == 1: majmin = version else: majmin = versions[0] + '.' + versions[1] url = 'http://download.gnome.org/sources/%s/%s/%s-%s.tar.xz' % (name, majmin, name, version) else: print >> sys.stderr, 'Unknown upstream group for metadata: %s (full line: \'%s\').' % (match.group(1), line) url = '' ignore = False if real_upstream_data.has_key(name): (current_version, current_url) = real_upstream_data[name] if util.version_ge(current_version, version): ignore = True if not ignore: real_upstream_data[name] = (version, url) # Now also fill data for 'glib|1.2.10' if it fits if upstream_name_branches.has_key(name): # name = 'glib', upstream_name_branch = 'glib|1.2.10' # and limit = '1.2.10' for (upstream_name_branch, limit) in upstream_name_branches[name]: if real_upstream_data.has_key(upstream_name_branch): (current_version, current_url) = real_upstream_data[upstream_name_branch] if util.version_ge(current_version, version): continue if util.version_ge(version, limit): continue real_upstream_data[upstream_name_branch] = (version, url) for (name, (version, url)) in real_upstream_data.items(): if olddata.has_key(name): # Update the entry if it has changed (id, oldversion, oldurl) = olddata[name] if oldversion != version or oldurl != url: # Note: we don't put the mtime here, since we use the # updated time in get_changed_packages self.cursor.execute('''UPDATE upstream SET version = ?, url = ?, updated = ? WHERE id = ? ;''', (version, url, self._now, id)) del olddata[name] else: # Add the entry self.cursor.execute('''INSERT INTO upstream VALUES ( NULL, ?, ?, ?, ?, ? );''', (branch_id, name, version, url, self._now)) file.close() # Remove data that was removed in the source file if len(olddata) > 0: ids = [ id for (id, version, url) in olddata.values() ] # Delete by group of 50, since it once had to remove ~1800 items # and it didn't work fine chunk_size = 50 ids_len = len(ids) for index in range(ids_len / chunk_size): chunk_ids = ids[index * chunk_size : (index + 1) * chunk_size] where = ' OR '.join([ 'id = ?' for i in range(len(chunk_ids)) ]) self.cursor.execute('''DELETE FROM upstream WHERE %s;''' % where, chunk_ids) remainder = ids_len % chunk_size if remainder > 0: chunk_ids = ids[- remainder:] where = ' OR '.join([ 'id = ?' for i in range(len(chunk_ids)) ]) self.cursor.execute('''DELETE FROM upstream WHERE %s;''' % where, chunk_ids) self._removed_upstream[branch] = olddata.keys() else: self._removed_upstream[branch] = []
os.makedirs(dirname) except OSError, e: if e.errno != errno.EEXIST: raise e if expected_hash_type and expected_hash: old_cache = os.path.join(self._cachedir, subpath) old_hash = get_hash_from_file(expected_hash_type, old_cache) if old_hash == expected_hash: shutil.copy(old_cache, cache) return fout = open(cache, 'w') fin = None try: fin = urllib2.urlopen(posixjoin(self.resource, subpath)) while True: bytes = fin.read(500 * 1024) if len(bytes) == 0: break fout.write(bytes) fout.close() except urllib2.HTTPError, e: if fin: fin.close() fout.close() os.unlink(cache) raise e fin.close() fout.close()
def put_object(self, data_source, object_name): dest_path = object_remote_location(object_name) self.writer.folder_ensure(dest_path[0]) self.writer.put(data_source, posixjoin(*dest_path))
def __init__(self, url_base, subrepo='', verbose=0): self._verbose = verbose self._base = posixjoin(url_base, subrepo) if not self._base.endswith('/'): self._base += '/'