def download(self): utils.debug(1, "downloading %s@%s tarball from npm registry" % (self.name, self.version)) info = _getstatusoutput('npm pack "%s@%s"' % (self.name, self.version)) if info[0] is not 0: exception = "Error downloading package %s@%s\n" % (self.name, self.version) exception += info[1] raise ValueError(exception) tarball_file = info[1].split('\n')[-1] tarball = tarfile.open(tarball_file) # get the root directory name root_dir = tarball.getnames()[0] # extract root directory name if memberfile contains '/' index_of_slash = root_dir.find('/') if index_of_slash != -1: root_dir = root_dir[:index_of_slash] tarball.extractall() tarball.close() # remove tarball file _os.remove(tarball_file) if root_dir is not self.debian_name: utils.debug(2, "renaming %s to %s" % (root_dir, self.debian_name)) # rename extracted directory _os.rename(root_dir, self.debian_name)
def search_for_repository(module): if isinstance(module, Npm2Deb): module = module.name repositories = ['collab-maint', 'pkg-javascript'] formatted = " {0:40} -- {1}" found = False result = {} my_print("Looking for existing repositories:") for repo in repositories: debug(1, "search for %s in %s" % (module, repo)) url_base = "http://anonscm.debian.org/gitweb" data = urlopen("%s/?a=project_list&pf=%s&s=%s" % (url_base, repo, module)).read() dom = minidom.parseString(data) for row in dom.getElementsByTagName('tr')[1:]: try: columns = row.getElementsByTagName('td') name = columns[0].firstChild.getAttribute('href')\ .split('.git')[0].split('?p=')[1] description = columns[1].firstChild.getAttribute('title') found = True result[name] = description my_print(formatted.format(name, description)) except: continue if not found: my_print(" None") return result
def _get_json_repo_url(self): result = 'FIX_ME repo url' url = None if 'repository' in self.json: repository = self.json['repository'] if isinstance(repository, str): url = repository elif isinstance(repository, dict) and 'url' in repository: url = repository['url'] if not url: pass # repository field is not in expected format elif url.startswith('git') or (isinstance(repository, dict) and 'type' in repository and repository['type'] == 'git'): if url.find('github') >= 0: tmp = self._get_github_url_from_git(url) if tmp: result = tmp else: utils.debug(1, ("raw URL: %s" % url)) url = _re.sub(r'^git.*?@(.*?):', r'https://\1/', url) url = _re.sub(r'^.*?(?:ssh|git).*?://(?:.*?\@)?', 'https://', url) url = _re.sub(r'\.git$', '', url) result = url utils.debug(1, ("https URL: %s" % url)) else: result = url self.upstream_repo_url = result
def download(self): utils.debug( 1, "downloading %s@%s tarball from npm registry" % (self.name, self.version)) info = _getstatusoutput('npm pack "%s@%s"' % (self.name, self.version)) if info[0] is not 0: exception = "Error downloading package %s@%s\n" % (self.name, self.version) exception += info[1] raise ValueError(exception) tarball_file = info[1].split('\n')[-1] tarball = tarfile.open(tarball_file) # get the root directory name root_dir = tarball.getnames()[0] # extract root directory name if memberfile contains '/' index_of_slash = root_dir.find('/') if index_of_slash != -1: root_dir = root_dir[:index_of_slash] tarball.extractall() tarball.close() # remove tarball file _os.remove(tarball_file) if root_dir is not self.debian_name: utils.debug(2, "renaming %s to %s" % (root_dir, self.debian_name)) # rename extracted directory _os.rename(root_dir, self.debian_name)
def clean(self): utils.debug(1, "cleaning directory") for filename in os.listdir("."): if filename != "debian": if os.path.isdir(filename): rmtree(filename) else: os.remove(filename)
def clean(self): utils.debug(1, "cleaning directory") for filename in _os.listdir('.'): if filename != 'debian': if _os.path.isdir(filename): _rmtree(filename) else: _os.remove(filename)
def __init__(self): if self.INSTANCE is not None: raise ValueError("Mapper is a Singleton. " \ "Please use get_instance method.") debug(2, 'loading database from %s' % DB_URL) data = findall('{{{(.*)}}}', \ urlopen("%s?action=raw" % DB_URL).read().replace('\n', ''))[0] self.json = parseJSON(data) self._warnings = {} self.reset_warnings()
def download_tarball(self): utils.debug( 1, "downloading %s@%s tarball from npm registry" % (self.name, self.version)) info = _getstatusoutput('npm pack "%s@%s"' % (self.name, self.version)) if info[0] is not 0: exception = "Error downloading package %s@%s\n" % (self.name, self.version) exception += info[1] raise ValueError(exception) tarball_file = info[1].split('\n')[-1] self.extract_tarball(tarball_file)
def download(self): utils.debug(1, "downloading %s via npm" % self.name) info = getstatusoutput("npm install %s" % self.name) if info[0] is not 0: exception = "Error downloading package %s\n" % self.name exception += info[1] raise ValueError(exception) # move dir from node_modules os.rename("node_modules/%s" % self.name, self.name) rmtree("node_modules") # remove any dependency downloaded via npm if os.path.isdir("%s/node_modules" % self.name): rmtree("%s/node_modules" % self.name) if self.name is not self.debian_name: utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name)) os.rename(self.name, self.debian_name)
def download(self): utils.debug(1, "downloading %s via npm" % self.name) info = getstatusoutput('npm install %s' % self.name) if info[0] is not 0: exception = "Error downloading package %s\n" % self.name exception += info[1] raise ValueError(exception) # move dir from node_modules os.rename('node_modules/%s' % self.name, self.name) rmtree('node_modules') # remove any dependency downloaded via npm if os.path.isdir("%s/node_modules" % self.name): rmtree("%s/node_modules" % self.name) if self.name is not self.debian_name: utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name)) os.rename(self.name, self.debian_name)
def search_for_reverse_dependencies(module): if isinstance(module, Npm2Deb): module = module.name url = "http://registry.npmjs.org/-/_view/dependedUpon?startkey=" \ + "[%%22%(name)s%%22]&endkey=[%%22%(name)s%%22,%%7B%%7D]&group_level=2" url = url % {'name': module} debug(1, "opening url %s" % url) data = urlopen(url).read() data = parseJSON(data) result = [] if 'rows' in data and len(data['rows']) > 0: my_print("Reverse Depends:") for row in data['rows']: dependency = row['key'][1] result.append(dependency) my_print(" %s" % dependency) else: my_print("Module %s has no reverse dependencies" % module) return result
def download(self): utils.debug(1, "downloading %s via npm" % self.name) info = _getstatusoutput('npm install "%s"' % self.name) if info[0] is not 0: exception = "Error downloading package %s\n" % self.name exception += info[1] raise ValueError(exception) # move dir from npm root root = _getstatusoutput('npm root')[1].strip('\n') _os.rename(_os.path.join(root, self.name), self.name) try: _os.rmdir(root) # remove only if empty except OSError: pass # remove any dependency downloaded via npm if _os.path.isdir("%s/node_modules" % self.name): _rmtree("%s/node_modules" % self.name) if self.name is not self.debian_name: utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name)) _os.rename(self.name, self.debian_name)
def download(self): utils.debug(1, "downloading %s tarball from npm registry" % self.name) info = _getstatusoutput('npm pack "%s"' % self.name) if info[0] is not 0: exception = "Error downloading package %s\n" % self.name exception += info[1] raise ValueError(exception) tarball_file = info[1].strip('\n') tarball = tarfile.open(tarball_file) tarball.extractall() tarball.close() # rename extracted directory _os.rename('package', self.name) # remove tarball file _os.remove(tarball_file) if self.name is not self.debian_name: utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name)) _os.rename(self.name, self.debian_name)
def create_watch(self): args = {} args['debian_name'] = self.debian_name args['dversionmangle'] = 'auto' args['url'] = self.upstream_repo_url args['module'] = self.name args['remodule'] = _re.sub(r'\@', r'\@', self.name) args['modulename'] = _re.sub(r'.*/', r'', self.name) try: if self.upstream_repo_url.find('github') >= 0: utils.debug(1, 'Found GitHub url') content = utils.get_watch('github') % args elif self.upstream_repo_url.find('gitlab') >= 0: utils.debug(1, 'Found GitLab url') content = utils.get_watch('gitlab') % args else: # if not supported, got to npmregistry raise ValueError utils.create_debian_file('watch', content) # test watch with uscan, raise exception if status is not 0 uscan_info = self.test_uscan() if uscan_info[0] != 0: utils.debug(1, 'Uscan failed, switching to npmregistry') raise ValueError except ValueError: self.upstream_watch = True content = utils.get_watch('npmregistry') % args utils.create_debian_file('watch', content)
def read_package_info(self): utils.debug(1, "reading package info using npm view") info = getstatusoutput('npm view %s --json 2>/dev/null' % self.name) # if not status 0, raise expection if info[0] != 0: info = getstatusoutput('npm view %s --json' % self.name) exception = 'npm reports errors about %s module:\n' % self.name exception += info[1] raise ValueError(exception) if not info[1]: exception = 'npm returns empty json for %s module' % self.name raise ValueError(exception) try: self.json = parseJSON(info[1]) except ValueError as value_error: # if error during parse, try to fail graceful if str(value_error) == 'No JSON object could be decoded': versions = [] for line in info[1].split('\n'): if re.match(r"^[a-z](.*)@[0-9]", line): version = line.split('@')[1] versions.append(version) if len(versions) > 0: exception = "More than one version found. "\ "Please specify one of:\n %s" % '\n '.join(versions) raise ValueError(exception) else: raise value_error else: raise value_error self.name = self.json['name'] self._get_json_author() self._get_json_repo_url() self._get_json_homepage() self._get_json_description() self._get_json_version() self._get_json_license()
def read_package_info(self): data = None name_is = None if _re.match("^(http:\/\/|https:\/\/)", self.name): utils.debug(1, "reading json - opening url %s" % self.name) data = _urlopen(self.name).read().decode('utf-8') name_is = 'url' elif _os.path.isfile(self.name): utils.debug(1, "reading json - opening file %s" % self.name) with open(self.name, 'r') as fd: data = fd.read() name_is = 'file' else: name_is = 'npm' utils.debug(1, "reading json - calling npm view %s" % self.name) info = _getstatusoutput('npm view "%s" --json 2>/dev/null' % self.name) # if not status 0, raise expection if info[0] != 0: info = _getstatusoutput('npm view "%s" --json' % self.name) exception = 'npm reports errors about %s module:\n' % self.name exception += info[1] raise ValueError(exception) if not info[1]: exception = 'npm returns empty json for %s module' % self.name raise ValueError(exception) data = info[1] try: self.json = _parseJSON(data) except ValueError as value_error: # if error during parse, try to fail graceful if str(value_error) == 'Expecting value: line 1 column 1 (char 0)': if name_is != 'npm': raise ValueError("Data read from %s " "is not in a JSON format." % name_is) versions = [] for line in data.split('\n'): if _re.match(r"^[a-z](.*)@[0-9]", line): version = line.split('@')[1] versions.append(version) if len(versions) > 0: exception = "More than one version found. "\ "Please specify one of:\n %s" % '\n '.join(versions) raise ValueError(exception) else: raise value_error else: raise value_error self.name = self.json['name'] self._get_json_author() self._get_json_repo_url() self._get_json_homepage() self._get_json_description() self._get_json_version() self._get_json_license()
def read_package_info(self): utils.debug(1, "reading package info using npm view") info = getstatusoutput("npm view %s --json 2>/dev/null" % self.name) # if not status 0, raise expection if info[0] != 0: info = getstatusoutput("npm view %s --json" % self.name) exception = "npm reports errors about %s module:\n" % self.name exception += info[1] raise ValueError(exception) if not info[1]: exception = "npm returns empty json for %s module" % self.name raise ValueError(exception) try: self.json = parseJSON(info[1]) except ValueError as value_error: # if error during parse, try to fail graceful if str(value_error) == "No JSON object could be decoded": versions = [] for line in info[1].split("\n"): if re.match(r"^[a-z](.*)@[0-9]", line): version = line.split("@")[1] versions.append(version) if len(versions) > 0: exception = "More than one version found. " "Please specify one of:\n %s" % "\n ".join(versions) raise ValueError(exception) else: raise value_error else: raise value_error self.name = self.json["name"] self._get_json_author() self._get_json_repo_url() self._get_json_homepage() self._get_json_description() self._get_json_version() self._get_json_license()
def search_for_bug(module): if isinstance(module, Npm2Deb): module = module.name url = 'http://wnpp.debian.net/' \ '?type%5B%5D=ITA&type%5B%5D=ITP&type%5B%5D=O&type%5B%5D=RFA' \ '&type%5B%5D=RFH&type%5B%5D=RFP&project=&description=&owner%5B%5D=yes' \ '&owner%5B%5D=no&col%5B%5D=type&col%5B%5D=description&sort=project' my_print('Looking for wnpp bugs:') debug(1, "getting bugs from http://wnpp.debian.net") data = urlopen(url).read() rows = data.split('class="data_table">')[1].split('</table>')[0].split( '\n') found = False formatted = " #{0} {1:>3}: {2:25} -- {3}" result = [] for row in rows: row = row.replace(' ', '') if len(row) > 0: try: bug = {} dom = minidom.parseString(row) tmp = dom.getElementsByTagName('a')[0] bug["num"] = tmp.getAttribute('href').split('?bug=')[1] bug["package"] = tmp.childNodes[0].data tmp = dom.getElementsByTagName('td')[-1] bug["type"] = tmp.getAttribute('class') bug["description"] = tmp.childNodes[0].childNodes[0].data if bug["package"].find(module) >= 0 or \ bug["description"].find(module) >= 0: found = True result.append(bug) my_print(formatted.format(bug["num"], \ bug["type"], bug["package"], bug["description"])) except: continue if not found: my_print(' None') return result
def search_for_bug(module): if isinstance(module, Npm2Deb): module = module.name url = 'http://wnpp.debian.net/' \ '?type%5B%5D=ITA&type%5B%5D=ITP&type%5B%5D=O&type%5B%5D=RFA' \ '&type%5B%5D=RFH&type%5B%5D=RFP&project=&description=&owner%5B%5D=yes' \ '&owner%5B%5D=no&col%5B%5D=type&col%5B%5D=description&sort=project' my_print('Looking for wnpp bugs:') debug(1, "getting bugs from http://wnpp.debian.net") data = urlopen(url).read() rows = data.split('class="data_table">')[1].split('</table>')[0].split('\n') found = False formatted = " #{0} {1:>3}: {2:25} -- {3}" result = [] for row in rows: row = row.replace(' ', '') if len(row) > 0: try: bug = {} dom = minidom.parseString(row) tmp = dom.getElementsByTagName('a')[0] bug["num"] = tmp.getAttribute('href').split('?bug=')[1] bug["package"] = tmp.childNodes[0].data tmp = dom.getElementsByTagName('td')[-1] bug["type"] = tmp.getAttribute('class') bug["description"] = tmp.childNodes[0].childNodes[0].data if bug["package"].find(module) >= 0 or \ bug["description"].find(module) >= 0: found = True result.append(bug) my_print(formatted.format(bug["num"], \ bug["type"], bug["package"], bug["description"])) except: continue if not found: my_print(' None') return result
def extract_tarball(self, tarball_file): utils.debug(2, "extracting tarball....") tarball = tarfile.open(tarball_file) # get the root directory name root_dir = tarball.getnames()[0] # extract root directory name if memberfile contains '/' index_of_slash = root_dir.find('/') if index_of_slash != -1: root_dir = root_dir[:index_of_slash] tarball.extractall() tarball.close() # remove tarball file if not self.noclean: _os.remove(tarball_file) if root_dir is not self.debian_name: utils.debug(2, "renaming %s to %s" % (root_dir, self.debian_name)) # rename extracted directory _os.rename(root_dir, self.debian_name) utils.debug(3, "tarball extracted")
def given_tarball(self): utils.debug(1, "opening %s tarball" % (self.module_name)) tarball_file = "../" + self.module_name self.extract_tarball(tarball_file)
def create_base_debian(self): utils.debug(1, "creating debian files") utils.create_dir("debian") utils.create_dir("debian/source") utils.create_debian_file("source/format", "3.0 (quilt)\n")
def create_base_debian(self): utils.debug(1, "creating debian files") utils.create_dir("debian") utils.create_dir("debian/source") utils.create_debian_file("source/format", "3.0 (quilt)\n") utils.create_debian_file("compat", self.debian_debhelper)
def create_itp_bug(self): utils.debug(1, "creating wnpp bug template") utils.create_file('%s_itp.mail' % self.debian_name, self.get_ITP())
def create_itp_bug(self): utils.debug(1, "creating wnpp bug template") utils.create_file("%s_itp.mail" % self.debian_name, self._get_ITP())