def download(self):
utils.debug(1, "downloading %s@%s tarball from npm registry" % (self.name, self.version))
info = _getstatusoutput('npm pack "%s@%s"' % (self.name, self.version))
if info[0] is not 0:
exception = "Error downloading package %s@%s\n" % (self.name, self.version)
exception += info[1]
raise ValueError(exception)
tarball_file = info[1].split('\n')[-1]
tarball = tarfile.open(tarball_file)
# get the root directory name
root_dir = tarball.getnames()[0]
# extract root directory name if memberfile contains '/'
index_of_slash = root_dir.find('/')
if index_of_slash != -1:
root_dir = root_dir[:index_of_slash]
tarball.extractall()
tarball.close()
# remove tarball file
_os.remove(tarball_file)
if root_dir is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (root_dir, self.debian_name))
# rename extracted directory
_os.rename(root_dir, self.debian_name)
def search_for_repository(module):
if isinstance(module, Npm2Deb):
module = module.name
repositories = ['collab-maint', 'pkg-javascript']
formatted = " {0:40} -- {1}"
found = False
result = {}
my_print("Looking for existing repositories:")
for repo in repositories:
debug(1, "search for %s in %s" % (module, repo))
url_base = "http://anonscm.debian.org/gitweb"
data = urlopen("%s/?a=project_list&pf=%s&s=%s" %
(url_base, repo, module)).read()
dom = minidom.parseString(data)
for row in dom.getElementsByTagName('tr')[1:]:
try:
columns = row.getElementsByTagName('td')
name = columns[0].firstChild.getAttribute('href')\
.split('.git')[0].split('?p=')[1]
description = columns[1].firstChild.getAttribute('title')
found = True
result[name] = description
my_print(formatted.format(name, description))
except:
continue
if not found:
my_print(" None")
return result
def search_for_repository(module):
if isinstance(module, Npm2Deb):
module = module.name
repositories = ['collab-maint', 'pkg-javascript']
formatted = " {0:40} -- {1}"
found = False
result = {}
my_print("Looking for existing repositories:")
for repo in repositories:
debug(1, "search for %s in %s" % (module, repo))
url_base = "http://anonscm.debian.org/gitweb"
data = urlopen("%s/?a=project_list&pf=%s&s=%s" %
(url_base, repo, module)).read()
dom = minidom.parseString(data)
for row in dom.getElementsByTagName('tr')[1:]:
try:
columns = row.getElementsByTagName('td')
name = columns[0].firstChild.getAttribute('href')\
.split('.git')[0].split('?p=')[1]
description = columns[1].firstChild.getAttribute('title')
found = True
result[name] = description
my_print(formatted.format(name, description))
except:
continue
if not found:
my_print(" None")
return result
def _get_json_repo_url(self):
result = 'FIX_ME repo url'
url = None
if 'repository' in self.json:
repository = self.json['repository']
if isinstance(repository, str):
url = repository
elif isinstance(repository, dict) and 'url' in repository:
url = repository['url']
if not url:
pass # repository field is not in expected format
elif url.startswith('git') or (isinstance(repository, dict)
and 'type' in repository
and repository['type'] == 'git'):
if url.find('github') >= 0:
tmp = self._get_github_url_from_git(url)
if tmp:
result = tmp
else:
utils.debug(1, ("raw URL: %s" % url))
url = _re.sub(r'^git.*?@(.*?):', r'https://\1/', url)
url = _re.sub(r'^.*?(?:ssh|git).*?://(?:.*?\@)?', 'https://', url)
url = _re.sub(r'\.git$', '', url)
result = url
utils.debug(1, ("https URL: %s" % url))
else:
result = url
self.upstream_repo_url = result
def download(self):
utils.debug(
1, "downloading %s@%s tarball from npm registry" %
(self.name, self.version))
info = _getstatusoutput('npm pack "%s@%s"' % (self.name, self.version))
if info[0] is not 0:
exception = "Error downloading package %s@%s\n" % (self.name,
self.version)
exception += info[1]
raise ValueError(exception)
tarball_file = info[1].split('\n')[-1]
tarball = tarfile.open(tarball_file)
# get the root directory name
root_dir = tarball.getnames()[0]
# extract root directory name if memberfile contains '/'
index_of_slash = root_dir.find('/')
if index_of_slash != -1:
root_dir = root_dir[:index_of_slash]
tarball.extractall()
tarball.close()
# remove tarball file
_os.remove(tarball_file)
if root_dir is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (root_dir, self.debian_name))
# rename extracted directory
_os.rename(root_dir, self.debian_name)
def _get_json_repo_url(self):
result = 'FIX_ME repo url'
url = None
if 'repository' in self.json:
repository = self.json['repository']
if isinstance(repository, str):
url = repository
elif isinstance(repository, dict) and 'url' in repository:
url = repository['url']
if not url:
pass # repository field is not in expected format
elif url.startswith('git') or (isinstance(repository, dict)
and 'type' in repository
and repository['type'] == 'git'):
if url.find('github') >= 0:
tmp = self._get_github_url_from_git(url)
if tmp:
result = tmp
else:
utils.debug(1, ("raw URL: %s" % url))
url = _re.sub(r'^git.*?@(.*?):', r'https://\1/', url)
url = _re.sub(r'^.*?(?:ssh|git).*?://(?:.*?\@)?',
'https://', url)
url = _re.sub(r'\.git$', '', url)
result = url
utils.debug(1, ("https URL: %s" % url))
else:
result = url
self.upstream_repo_url = result
def clean(self):
utils.debug(1, "cleaning directory")
for filename in os.listdir("."):
if filename != "debian":
if os.path.isdir(filename):
rmtree(filename)
else:
os.remove(filename)
def clean(self):
utils.debug(1, "cleaning directory")
for filename in _os.listdir('.'):
if filename != 'debian':
if _os.path.isdir(filename):
_rmtree(filename)
else:
_os.remove(filename)
def clean(self):
utils.debug(1, "cleaning directory")
for filename in _os.listdir('.'):
if filename != 'debian':
if _os.path.isdir(filename):
_rmtree(filename)
else:
_os.remove(filename)
def __init__(self):
if self.INSTANCE is not None:
raise ValueError("Mapper is a Singleton. " \
"Please use get_instance method.")
debug(2, 'loading database from %s' % DB_URL)
data = findall('{{{(.*)}}}', \
urlopen("%s?action=raw" % DB_URL).read().replace('\n', ''))[0]
self.json = parseJSON(data)
self._warnings = {}
self.reset_warnings()
def download_tarball(self):
utils.debug(
1, "downloading %s@%s tarball from npm registry" %
(self.name, self.version))
info = _getstatusoutput('npm pack "%s@%s"' % (self.name, self.version))
if info[0] is not 0:
exception = "Error downloading package %s@%s\n" % (self.name,
self.version)
exception += info[1]
raise ValueError(exception)
tarball_file = info[1].split('\n')[-1]
self.extract_tarball(tarball_file)
def download(self):
utils.debug(1, "downloading %s via npm" % self.name)
info = getstatusoutput("npm install %s" % self.name)
if info[0] is not 0:
exception = "Error downloading package %s\n" % self.name
exception += info[1]
raise ValueError(exception)
# move dir from node_modules
os.rename("node_modules/%s" % self.name, self.name)
rmtree("node_modules")
# remove any dependency downloaded via npm
if os.path.isdir("%s/node_modules" % self.name):
rmtree("%s/node_modules" % self.name)
if self.name is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name))
os.rename(self.name, self.debian_name)
def download(self):
utils.debug(1, "downloading %s via npm" % self.name)
info = getstatusoutput('npm install %s' % self.name)
if info[0] is not 0:
exception = "Error downloading package %s\n" % self.name
exception += info[1]
raise ValueError(exception)
# move dir from node_modules
os.rename('node_modules/%s' % self.name, self.name)
rmtree('node_modules')
# remove any dependency downloaded via npm
if os.path.isdir("%s/node_modules" % self.name):
rmtree("%s/node_modules" % self.name)
if self.name is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name))
os.rename(self.name, self.debian_name)
def search_for_reverse_dependencies(module):
if isinstance(module, Npm2Deb):
module = module.name
url = "http://registry.npmjs.org/-/_view/dependedUpon?startkey=" \
+ "[%%22%(name)s%%22]&endkey=[%%22%(name)s%%22,%%7B%%7D]&group_level=2"
url = url % {'name': module}
debug(1, "opening url %s" % url)
data = urlopen(url).read()
data = parseJSON(data)
result = []
if 'rows' in data and len(data['rows']) > 0:
my_print("Reverse Depends:")
for row in data['rows']:
dependency = row['key'][1]
result.append(dependency)
my_print(" %s" % dependency)
else:
my_print("Module %s has no reverse dependencies" % module)
return result
def search_for_reverse_dependencies(module):
if isinstance(module, Npm2Deb):
module = module.name
url = "http://registry.npmjs.org/-/_view/dependedUpon?startkey=" \
+ "[%%22%(name)s%%22]&endkey=[%%22%(name)s%%22,%%7B%%7D]&group_level=2"
url = url % {'name': module}
debug(1, "opening url %s" % url)
data = urlopen(url).read()
data = parseJSON(data)
result = []
if 'rows' in data and len(data['rows']) > 0:
my_print("Reverse Depends:")
for row in data['rows']:
dependency = row['key'][1]
result.append(dependency)
my_print(" %s" % dependency)
else:
my_print("Module %s has no reverse dependencies" % module)
return result
def download(self):
utils.debug(1, "downloading %s via npm" % self.name)
info = _getstatusoutput('npm install "%s"' % self.name)
if info[0] is not 0:
exception = "Error downloading package %s\n" % self.name
exception += info[1]
raise ValueError(exception)
# move dir from npm root
root = _getstatusoutput('npm root')[1].strip('\n')
_os.rename(_os.path.join(root, self.name), self.name)
try:
_os.rmdir(root) # remove only if empty
except OSError:
pass
# remove any dependency downloaded via npm
if _os.path.isdir("%s/node_modules" % self.name):
_rmtree("%s/node_modules" % self.name)
if self.name is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name))
_os.rename(self.name, self.debian_name)
def download(self):
utils.debug(1, "downloading %s via npm" % self.name)
info = _getstatusoutput('npm install "%s"' % self.name)
if info[0] is not 0:
exception = "Error downloading package %s\n" % self.name
exception += info[1]
raise ValueError(exception)
# move dir from npm root
root = _getstatusoutput('npm root')[1].strip('\n')
_os.rename(_os.path.join(root, self.name), self.name)
try:
_os.rmdir(root) # remove only if empty
except OSError:
pass
# remove any dependency downloaded via npm
if _os.path.isdir("%s/node_modules" % self.name):
_rmtree("%s/node_modules" % self.name)
if self.name is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name))
_os.rename(self.name, self.debian_name)
def download(self):
utils.debug(1, "downloading %s tarball from npm registry" % self.name)
info = _getstatusoutput('npm pack "%s"' % self.name)
if info[0] is not 0:
exception = "Error downloading package %s\n" % self.name
exception += info[1]
raise ValueError(exception)
tarball_file = info[1].strip('\n')
tarball = tarfile.open(tarball_file)
tarball.extractall()
tarball.close()
# rename extracted directory
_os.rename('package', self.name)
# remove tarball file
_os.remove(tarball_file)
if self.name is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name))
_os.rename(self.name, self.debian_name)
def create_watch(self):
args = {}
args['debian_name'] = self.debian_name
args['dversionmangle'] = 'auto'
args['url'] = self.upstream_repo_url
args['module'] = self.name
args['remodule'] = _re.sub(r'\@', r'\@', self.name)
args['modulename'] = _re.sub(r'.*/', r'', self.name)
try:
if self.upstream_repo_url.find('github') >= 0:
utils.debug(1, 'Found GitHub url')
content = utils.get_watch('github') % args
elif self.upstream_repo_url.find('gitlab') >= 0:
utils.debug(1, 'Found GitLab url')
content = utils.get_watch('gitlab') % args
else:
# if not supported, got to npmregistry
raise ValueError
utils.create_debian_file('watch', content)
# test watch with uscan, raise exception if status is not 0
uscan_info = self.test_uscan()
if uscan_info[0] != 0:
utils.debug(1, 'Uscan failed, switching to npmregistry')
raise ValueError
except ValueError:
self.upstream_watch = True
content = utils.get_watch('npmregistry') % args
utils.create_debian_file('watch', content)
def create_watch(self):
args = {}
args['debian_name'] = self.debian_name
args['dversionmangle'] = 'auto'
args['url'] = self.upstream_repo_url
args['module'] = self.name
args['remodule'] = _re.sub(r'\@', r'\@', self.name)
args['modulename'] = _re.sub(r'.*/', r'', self.name)
try:
if self.upstream_repo_url.find('github') >= 0:
utils.debug(1, 'Found GitHub url')
content = utils.get_watch('github') % args
elif self.upstream_repo_url.find('gitlab') >= 0:
utils.debug(1, 'Found GitLab url')
content = utils.get_watch('gitlab') % args
else:
# if not supported, got to npmregistry
raise ValueError
utils.create_debian_file('watch', content)
# test watch with uscan, raise exception if status is not 0
uscan_info = self.test_uscan()
if uscan_info[0] != 0:
utils.debug(1, 'Uscan failed, switching to npmregistry')
raise ValueError
except ValueError:
self.upstream_watch = True
content = utils.get_watch('npmregistry') % args
utils.create_debian_file('watch', content)
def read_package_info(self):
utils.debug(1, "reading package info using npm view")
info = getstatusoutput('npm view %s --json 2>/dev/null' % self.name)
# if not status 0, raise expection
if info[0] != 0:
info = getstatusoutput('npm view %s --json' % self.name)
exception = 'npm reports errors about %s module:\n' % self.name
exception += info[1]
raise ValueError(exception)
if not info[1]:
exception = 'npm returns empty json for %s module' % self.name
raise ValueError(exception)
try:
self.json = parseJSON(info[1])
except ValueError as value_error:
# if error during parse, try to fail graceful
if str(value_error) == 'No JSON object could be decoded':
versions = []
for line in info[1].split('\n'):
if re.match(r"^[a-z](.*)@[0-9]", line):
version = line.split('@')[1]
versions.append(version)
if len(versions) > 0:
exception = "More than one version found. "\
"Please specify one of:\n %s" % '\n '.join(versions)
raise ValueError(exception)
else:
raise value_error
else:
raise value_error
self.name = self.json['name']
self._get_json_author()
self._get_json_repo_url()
self._get_json_homepage()
self._get_json_description()
self._get_json_version()
self._get_json_license()
def read_package_info(self):
data = None
name_is = None
if _re.match("^(http:\/\/|https:\/\/)", self.name):
utils.debug(1, "reading json - opening url %s" % self.name)
data = _urlopen(self.name).read().decode('utf-8')
name_is = 'url'
elif _os.path.isfile(self.name):
utils.debug(1, "reading json - opening file %s" % self.name)
with open(self.name, 'r') as fd:
data = fd.read()
name_is = 'file'
else:
name_is = 'npm'
utils.debug(1, "reading json - calling npm view %s" % self.name)
info = _getstatusoutput('npm view "%s" --json 2>/dev/null' %
self.name)
# if not status 0, raise expection
if info[0] != 0:
info = _getstatusoutput('npm view "%s" --json' % self.name)
exception = 'npm reports errors about %s module:\n' % self.name
exception += info[1]
raise ValueError(exception)
if not info[1]:
exception = 'npm returns empty json for %s module' % self.name
raise ValueError(exception)
data = info[1]
try:
self.json = _parseJSON(data)
except ValueError as value_error:
# if error during parse, try to fail graceful
if str(value_error) == 'Expecting value: line 1 column 1 (char 0)':
if name_is != 'npm':
raise ValueError("Data read from %s "
"is not in a JSON format." % name_is)
versions = []
for line in data.split('\n'):
if _re.match(r"^[a-z](.*)@[0-9]", line):
version = line.split('@')[1]
versions.append(version)
if len(versions) > 0:
exception = "More than one version found. "\
"Please specify one of:\n %s" % '\n '.join(versions)
raise ValueError(exception)
else:
raise value_error
else:
raise value_error
self.name = self.json['name']
self._get_json_author()
self._get_json_repo_url()
self._get_json_homepage()
self._get_json_description()
self._get_json_version()
self._get_json_license()
def read_package_info(self):
data = None
name_is = None
if _re.match("^(http:\/\/|https:\/\/)", self.name):
utils.debug(1, "reading json - opening url %s" % self.name)
data = _urlopen(self.name).read().decode('utf-8')
name_is = 'url'
elif _os.path.isfile(self.name):
utils.debug(1, "reading json - opening file %s" % self.name)
with open(self.name, 'r') as fd:
data = fd.read()
name_is = 'file'
else:
name_is = 'npm'
utils.debug(1, "reading json - calling npm view %s" % self.name)
info = _getstatusoutput('npm view "%s" --json 2>/dev/null' %
self.name)
# if not status 0, raise expection
if info[0] != 0:
info = _getstatusoutput('npm view "%s" --json' % self.name)
exception = 'npm reports errors about %s module:\n' % self.name
exception += info[1]
raise ValueError(exception)
if not info[1]:
exception = 'npm returns empty json for %s module' % self.name
raise ValueError(exception)
data = info[1]
try:
self.json = _parseJSON(data)
except ValueError as value_error:
# if error during parse, try to fail graceful
if str(value_error) == 'Expecting value: line 1 column 1 (char 0)':
if name_is != 'npm':
raise ValueError("Data read from %s "
"is not in a JSON format." % name_is)
versions = []
for line in data.split('\n'):
if _re.match(r"^[a-z](.*)@[0-9]", line):
version = line.split('@')[1]
versions.append(version)
if len(versions) > 0:
exception = "More than one version found. "\
"Please specify one of:\n %s" % '\n '.join(versions)
raise ValueError(exception)
else:
raise value_error
else:
raise value_error
self.name = self.json['name']
self._get_json_author()
self._get_json_repo_url()
self._get_json_homepage()
self._get_json_description()
self._get_json_version()
self._get_json_license()
def read_package_info(self):
utils.debug(1, "reading package info using npm view")
info = getstatusoutput("npm view %s --json 2>/dev/null" % self.name)
# if not status 0, raise expection
if info[0] != 0:
info = getstatusoutput("npm view %s --json" % self.name)
exception = "npm reports errors about %s module:\n" % self.name
exception += info[1]
raise ValueError(exception)
if not info[1]:
exception = "npm returns empty json for %s module" % self.name
raise ValueError(exception)
try:
self.json = parseJSON(info[1])
except ValueError as value_error:
# if error during parse, try to fail graceful
if str(value_error) == "No JSON object could be decoded":
versions = []
for line in info[1].split("\n"):
if re.match(r"^[a-z](.*)@[0-9]", line):
version = line.split("@")[1]
versions.append(version)
if len(versions) > 0:
exception = "More than one version found. " "Please specify one of:\n %s" % "\n ".join(versions)
raise ValueError(exception)
else:
raise value_error
else:
raise value_error
self.name = self.json["name"]
self._get_json_author()
self._get_json_repo_url()
self._get_json_homepage()
self._get_json_description()
self._get_json_version()
self._get_json_license()
def search_for_bug(module):
if isinstance(module, Npm2Deb):
module = module.name
url = 'http://wnpp.debian.net/' \
'?type%5B%5D=ITA&type%5B%5D=ITP&type%5B%5D=O&type%5B%5D=RFA' \
'&type%5B%5D=RFH&type%5B%5D=RFP&project=&description=&owner%5B%5D=yes' \
'&owner%5B%5D=no&col%5B%5D=type&col%5B%5D=description&sort=project'
my_print('Looking for wnpp bugs:')
debug(1, "getting bugs from http://wnpp.debian.net")
data = urlopen(url).read()
rows = data.split('class="data_table">')[1].split('</table>')[0].split(
'\n')
found = False
formatted = " #{0} {1:>3}: {2:25} -- {3}"
result = []
for row in rows:
row = row.replace(' ', '')
if len(row) > 0:
try:
bug = {}
dom = minidom.parseString(row)
tmp = dom.getElementsByTagName('a')[0]
bug["num"] = tmp.getAttribute('href').split('?bug=')[1]
bug["package"] = tmp.childNodes[0].data
tmp = dom.getElementsByTagName('td')[-1]
bug["type"] = tmp.getAttribute('class')
bug["description"] = tmp.childNodes[0].childNodes[0].data
if bug["package"].find(module) >= 0 or \
bug["description"].find(module) >= 0:
found = True
result.append(bug)
my_print(formatted.format(bug["num"], \
bug["type"], bug["package"], bug["description"]))
except:
continue
if not found:
my_print(' None')
return result
def search_for_bug(module):
if isinstance(module, Npm2Deb):
module = module.name
url = 'http://wnpp.debian.net/' \
'?type%5B%5D=ITA&type%5B%5D=ITP&type%5B%5D=O&type%5B%5D=RFA' \
'&type%5B%5D=RFH&type%5B%5D=RFP&project=&description=&owner%5B%5D=yes' \
'&owner%5B%5D=no&col%5B%5D=type&col%5B%5D=description&sort=project'
my_print('Looking for wnpp bugs:')
debug(1, "getting bugs from http://wnpp.debian.net")
data = urlopen(url).read()
rows = data.split('class="data_table">')[1].split('</table>')[0].split('\n')
found = False
formatted = " #{0} {1:>3}: {2:25} -- {3}"
result = []
for row in rows:
row = row.replace(' ', '')
if len(row) > 0:
try:
bug = {}
dom = minidom.parseString(row)
tmp = dom.getElementsByTagName('a')[0]
bug["num"] = tmp.getAttribute('href').split('?bug=')[1]
bug["package"] = tmp.childNodes[0].data
tmp = dom.getElementsByTagName('td')[-1]
bug["type"] = tmp.getAttribute('class')
bug["description"] = tmp.childNodes[0].childNodes[0].data
if bug["package"].find(module) >= 0 or \
bug["description"].find(module) >= 0:
found = True
result.append(bug)
my_print(formatted.format(bug["num"], \
bug["type"], bug["package"], bug["description"]))
except:
continue
if not found:
my_print(' None')
return result
def extract_tarball(self, tarball_file):
utils.debug(2, "extracting tarball....")
tarball = tarfile.open(tarball_file)
# get the root directory name
root_dir = tarball.getnames()[0]
# extract root directory name if memberfile contains '/'
index_of_slash = root_dir.find('/')
if index_of_slash != -1:
root_dir = root_dir[:index_of_slash]
tarball.extractall()
tarball.close()
# remove tarball file
if not self.noclean:
_os.remove(tarball_file)
if root_dir is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (root_dir, self.debian_name))
# rename extracted directory
_os.rename(root_dir, self.debian_name)
utils.debug(3, "tarball extracted")
def given_tarball(self):
utils.debug(1, "opening %s tarball" % (self.module_name))
tarball_file = "../" + self.module_name
self.extract_tarball(tarball_file)
def create_base_debian(self):
utils.debug(1, "creating debian files")
utils.create_dir("debian")
utils.create_dir("debian/source")
utils.create_debian_file("source/format", "3.0 (quilt)\n")
def create_base_debian(self):
utils.debug(1, "creating debian files")
utils.create_dir("debian")
utils.create_dir("debian/source")
utils.create_debian_file("source/format", "3.0 (quilt)\n")
utils.create_debian_file("compat", self.debian_debhelper)
def create_itp_bug(self):
utils.debug(1, "creating wnpp bug template")
utils.create_file('%s_itp.mail' % self.debian_name, self.get_ITP())
def create_itp_bug(self):
utils.debug(1, "creating wnpp bug template")
utils.create_file("%s_itp.mail" % self.debian_name, self._get_ITP())
