def read_package_info(self):
data = None
name_is = None
if _re.match("^(http:\/\/|https:\/\/)", self.name):
utils.debug(1, "reading json - opening url %s" % self.name)
data = _urlopen(self.name).read().decode('utf-8')
name_is = 'url'
elif _os.path.isfile(self.name):
utils.debug(1, "reading json - opening file %s" % self.name)
with open(self.name, 'r') as fd:
data = fd.read()
name_is = 'file'
else:
name_is = 'npm'
utils.debug(1, "reading json - calling npm view %s" % self.name)
info = _getstatusoutput('npm view "%s" --json 2>/dev/null' %
self.name)
# if not status 0, raise expection
if info[0] != 0:
info = _getstatusoutput('npm view "%s" --json' % self.name)
exception = 'npm reports errors about %s module:\n' % self.name
exception += info[1]
raise ValueError(exception)
if not info[1]:
exception = 'npm returns empty json for %s module' % self.name
raise ValueError(exception)
data = info[1]
try:
self.json = _parseJSON(data)
except ValueError as value_error:
# if error during parse, try to fail graceful
if str(value_error) == 'Expecting value: line 1 column 1 (char 0)':
if name_is != 'npm':
raise ValueError("Data read from %s "
"is not in a JSON format." % name_is)
versions = []
for line in data.split('\n'):
if _re.match(r"^[a-z](.*)@[0-9]", line):
version = line.split('@')[1]
versions.append(version)
if len(versions) > 0:
exception = "More than one version found. "\
"Please specify one of:\n %s" % '\n '.join(versions)
raise ValueError(exception)
else:
raise value_error
else:
raise value_error
self.name = self.json['name']
self._get_json_author()
self._get_json_repo_url()
self._get_json_homepage()
self._get_json_description()
self._get_json_version()
self._get_json_license()
def read_package_info(self):
data = None
name_is = None
if _re.match("^(http:\/\/|https:\/\/)", self.name):
utils.debug(1, "reading json - opening url %s" % self.name)
data = _urlopen(self.name).read().decode('utf-8')
name_is = 'url'
elif _os.path.isfile(self.name):
utils.debug(1, "reading json - opening file %s" % self.name)
with open(self.name, 'r') as fd:
data = fd.read()
name_is = 'file'
else:
name_is = 'npm'
utils.debug(1, "reading json - calling npm view %s" % self.name)
info = _getstatusoutput('npm view "%s" --json 2>/dev/null' %
self.name)
# if not status 0, raise expection
if info[0] != 0:
info = _getstatusoutput('npm view "%s" --json' % self.name)
exception = 'npm reports errors about %s module:\n' % self.name
exception += info[1]
raise ValueError(exception)
if not info[1]:
exception = 'npm returns empty json for %s module' % self.name
raise ValueError(exception)
data = info[1]
try:
self.json = _parseJSON(data)
except ValueError as value_error:
# if error during parse, try to fail graceful
if str(value_error) == 'Expecting value: line 1 column 1 (char 0)':
if name_is != 'npm':
raise ValueError("Data read from %s "
"is not in a JSON format." % name_is)
versions = []
for line in data.split('\n'):
if _re.match(r"^[a-z](.*)@[0-9]", line):
version = line.split('@')[1]
versions.append(version)
if len(versions) > 0:
exception = "More than one version found. "\
"Please specify one of:\n %s" % '\n '.join(versions)
raise ValueError(exception)
else:
raise value_error
else:
raise value_error
self.name = self.json['name']
self._get_json_author()
self._get_json_repo_url()
self._get_json_homepage()
self._get_json_description()
self._get_json_version()
self._get_json_license()
def search_for_builddep(module):
try:
if not isinstance(module, _Npm2Deb):
_debug(1, 'getting builddep - calling npm view %s' % module)
npm_out = _getstatusoutput('npm view "%s" '
'devDependencies --json 2>/dev/null'
% module)[1]
builddeb = _parseJSON(npm_out)
else:
builddeb = module.json['devDependencies']
module = module.name
except ValueError:
return None
mapper = _Mapper.get_instance()
result = {}
for dep in sorted(builddeb.keys()):
result[dep] = {}
result[dep]['version'] = builddeb[dep]
result[dep]['debian'] = mapper.get_debian_package(dep)['repr']
print_formatted_dependency("%s (%s)" % (dep, result[dep]['version']),
result[dep]['debian'])
return result
def search_for_bug(module):
if isinstance(module, _Npm2Deb):
module = module.name
my_print('Looking for wnpp bugs:')
_debug(1, "calling wnpp-check")
info = _getstatusoutput('wnpp-check %s' % module)
if info[0] == 0:
my_print(' None')
return []
else:
lines = info[1].split('\n')
formatted = " #{0} {1:>3}: {2:25} -- {3}"
result = []
for line in lines:
try:
bug = {}
match = _re.match('\((.*) - #(\d+).*\) (.*) (.*)$', line)
bug['num'] = match.group(2)
bug['type'] = match.group(1)
bug['package'] = match.group(4)
bug['url'] = match.group(3)
result.append(bug)
my_print(formatted.format(bug["num"],
bug["type"],
bug["package"],
bug["url"]))
except:
continue
return result
def create_watch(self):
args = {}
args['debian_name'] = self.debian_name
args['dversionmangle'] = 's/\+(debian|dfsg|ds|deb)(\.\d+)?$//'
args['url'] = self.upstream_repo_url
args['module'] = self.name
try:
if self.upstream_repo_url.find('github') >= 0:
content = utils.get_watch('github') % args
else:
# if not supported, got to fakeupstream
raise ValueError
utils.create_debian_file('watch', content)
# test watch with uscan, raise exception if status is not 0
info = _getstatusoutput(
'uscan --watchfile "debian/watch" '
'--package "{}" '
'--upstream-version 0 --no-download'.format(self.debian_name))
if info[0] != 0:
raise ValueError
except ValueError:
content = utils.get_watch('fakeupstream') % args
utils.create_debian_file('watch', content)
def search_for_repository(module):
if isinstance(module, _Npm2Deb):
module = module.name
repositories = ['collab-maint', 'pkg-javascript']
formatted = " {0:40} -- {1}"
found = False
result = {}
my_print("Looking for existing repositories:")
content = _getstatusoutput(["rmadison -u debian node-" + module])
if content[1]:
print(content[1]),
found = True
for repo in repositories:
_debug(1, "search for %s in %s" % (module, repo))
url_base = "http://anonscm.debian.org/gitweb"
data = _urlopen("%s/?a=project_list&pf=%s&s=%s" %
(url_base, repo, module)).read()
dom = _minidom.parseString(data)
for row in dom.getElementsByTagName('tr')[1:]:
try:
columns = row.getElementsByTagName('td')
name = columns[0].firstChild.getAttribute('href')\
.split('.git')[0].split('?p=')[1]
description = columns[1].firstChild.getAttribute('title')
found = True
result[name] = description
my_print(formatted.format(name, description))
except:
continue
if not found:
my_print(" None")
return result
def create_watch(self):
args = {}
args['debian_name'] = self.debian_name
args['dversionmangle'] = 's/\+(debian|dfsg|ds|deb)(\.\d+)?$//'
args['url'] = self.upstream_repo_url
args['module'] = self.name
try:
if self.upstream_repo_url.find('github') >= 0:
content = utils.get_watch('github') % args
else:
# if not supported, got to fakeupstream
raise ValueError
utils.create_debian_file('watch', content)
# test watch with uscan, raise exception if status is not 0
info = _getstatusoutput('uscan --watchfile "debian/watch" '
'--package "{}" '
'--upstream-version 0 --no-download'
.format(self.debian_name))
if info[0] != 0:
raise ValueError
except ValueError:
content = utils.get_watch('fakeupstream') % args
utils.create_debian_file('watch', content)
def download(self):
utils.debug(
1, "downloading %s@%s tarball from npm registry" %
(self.name, self.version))
info = _getstatusoutput('npm pack "%s@%s"' % (self.name, self.version))
if info[0] is not 0:
exception = "Error downloading package %s@%s\n" % (self.name,
self.version)
exception += info[1]
raise ValueError(exception)
tarball_file = info[1].split('\n')[-1]
tarball = tarfile.open(tarball_file)
# get the root directory name
root_dir = tarball.getnames()[0]
# extract root directory name if memberfile contains '/'
index_of_slash = root_dir.find('/')
if index_of_slash != -1:
root_dir = root_dir[:index_of_slash]
tarball.extractall()
tarball.close()
# remove tarball file
_os.remove(tarball_file)
if root_dir is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (root_dir, self.debian_name))
# rename extracted directory
_os.rename(root_dir, self.debian_name)
def search_for_builddep(module):
try:
if not isinstance(module, _Npm2Deb):
_debug(1, 'getting builddep - calling npm view %s' % module)
npm_out = _getstatusoutput('npm view "%s" '
'devDependencies --json 2>/dev/null' %
module)[1]
builddeb = _parseJSON(npm_out)
else:
builddeb = module.json['devDependencies']
module = module.name
except ValueError:
return None
mapper = _Mapper.get_instance()
result = {}
for dep in sorted(builddeb.keys()):
result[dep] = {}
result[dep]['version'] = builddeb[dep]
result[dep]['debian'] = mapper.get_debian_package(dep)['repr']
print_formatted_dependency("%s (%s)" % (dep, result[dep]['version']),
result[dep]['debian'])
return result
def download(self):
utils.debug(1, "downloading %s@%s tarball from npm registry" % (self.name, self.version))
info = _getstatusoutput('npm pack "%s@%s"' % (self.name, self.version))
if info[0] is not 0:
exception = "Error downloading package %s@%s\n" % (self.name, self.version)
exception += info[1]
raise ValueError(exception)
tarball_file = info[1].split('\n')[-1]
tarball = tarfile.open(tarball_file)
# get the root directory name
root_dir = tarball.getnames()[0]
# extract root directory name if memberfile contains '/'
index_of_slash = root_dir.find('/')
if index_of_slash != -1:
root_dir = root_dir[:index_of_slash]
tarball.extractall()
tarball.close()
# remove tarball file
_os.remove(tarball_file)
if root_dir is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (root_dir, self.debian_name))
# rename extracted directory
_os.rename(root_dir, self.debian_name)
def search_for_bug(module):
if isinstance(module, _Npm2Deb):
module = module.name
my_print('Looking for wnpp bugs:')
_debug(1, "calling wnpp-check")
info = _getstatusoutput('wnpp-check %s' % module)
if info[0] == 0:
my_print(' None')
return []
else:
lines = info[1].split('\n')
formatted = " #{0} {1:>3}: {2:25} -- {3}"
result = []
for line in lines:
try:
bug = {}
match = _re.match('\((.*) - #(\d+).*\) (.*) (.*)$', line)
bug['num'] = match.group(2)
bug['type'] = match.group(1)
bug['package'] = match.group(4)
bug['url'] = match.group(3)
result.append(bug)
my_print(
formatted.format(bug["num"], bug["type"], bug["package"],
bug["url"]))
except:
continue
return result
def test_uscan(self):
info = _getstatusoutput('uscan --watchfile "debian/watch" '
'--package "{}" '
'--upstream-version 0 '
'--download-version {} '
'--no-download'.format(self.debian_name,
self.upstream_version))
return info
def test_uscan(self):
info = _getstatusoutput('uscan --watchfile "debian/watch" '
'--package "{}" '
'--upstream-version 0 '
'--download-version {} '
'--no-download'.format(self.debian_name,
self.upstream_version))
return info
def _get_github_url_from_git(self, url):
result = _getstatusoutput(
"nodejs -e "
"\"console.log(require('github-url-from-git')"
"('%s'));\"" % url)[1]
if result == 'undefined':
result = None
return result
def _get_github_url_from_git(self, url):
result = _getstatusoutput(
"node -e "
"\"console.log(require('github-url-from-git')"
"('%s'));\"" % url)[1]
if result == 'undefined':
result = None
return result
def download(self):
utils.debug(1, "downloading %s via npm" % self.name)
info = _getstatusoutput('npm install "%s"' % self.name)
if info[0] is not 0:
exception = "Error downloading package %s\n" % self.name
exception += info[1]
raise ValueError(exception)
# move dir from npm root
root = _getstatusoutput('npm root')[1].strip('\n')
_os.rename(_os.path.join(root, self.name), self.name)
try:
_os.rmdir(root) # remove only if empty
except OSError:
pass
# remove any dependency downloaded via npm
if _os.path.isdir("%s/node_modules" % self.name):
_rmtree("%s/node_modules" % self.name)
if self.name is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name))
_os.rename(self.name, self.debian_name)
def download(self):
utils.debug(1, "downloading %s via npm" % self.name)
info = _getstatusoutput('npm install "%s"' % self.name)
if info[0] is not 0:
exception = "Error downloading package %s\n" % self.name
exception += info[1]
raise ValueError(exception)
# move dir from npm root
root = _getstatusoutput('npm root')[1].strip('\n')
_os.rename(_os.path.join(root, self.name), self.name)
try:
_os.rmdir(root) # remove only if empty
except OSError:
pass
# remove any dependency downloaded via npm
if _os.path.isdir("%s/node_modules" % self.name):
_rmtree("%s/node_modules" % self.name)
if self.name is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name))
_os.rename(self.name, self.debian_name)
def download_tarball(self):
utils.debug(
1, "downloading %s@%s tarball from npm registry" %
(self.name, self.version))
info = _getstatusoutput('npm pack "%s@%s"' % (self.name, self.version))
if info[0] is not 0:
exception = "Error downloading package %s@%s\n" % (self.name,
self.version)
exception += info[1]
raise ValueError(exception)
tarball_file = info[1].split('\n')[-1]
self.extract_tarball(tarball_file)
def get_debian_package(self, node_module):
result = {}
result['info'] = None
result['name'] = None
result['version'] = None
result['suite'] = None
result['repr'] = None
db_package = None
if node_module in self.json:
db_package = self.json[node_module]
else:
for pattern in self.json.keys():
if _fnmatch(node_module, pattern):
db_package = self.json[pattern]
break
if db_package:
if 'replace' in db_package:
result['name'] = db_package['replace']
if 'info' in db_package:
result['info'] = ('info', db_package['info'])
self.append_warning('info', node_module, db_package['info'])
elif 'warning' in db_package:
result['info'] = ('warning', db_package['warning'])
self.append_warning('warning', node_module,
db_package['warning'])
elif 'error' in db_package:
result['info'] = ('error', db_package['error'])
self.append_warning('error', node_module, db_package['error'])
else:
result['name'] = 'node-%s' % _debianize_name(node_module)
if not result['name']:
return result
madison = _getstatusoutput('rmadison -u debian "%s" | grep source' %
result['name'])
if madison[0] != 0:
result['name'] = None
return result
tmp = madison[1].split('|')
if len(tmp) >= 2:
result['name'] = tmp[0].strip()
result['version'] = tmp[1].strip()
result['suite'] = tmp[2].strip()
result['repr'] = '%s (%s)' % (result['name'], result['version'])
return result
def search_for_dependencies(module,
recursive=False,
force=False,
prefix=u'',
expanded_dependencies=[]):
try:
if not isinstance(module, _Npm2Deb):
_debug(1, 'getting dependencies - calling npm view %s' % module)
npm_out = _getstatusoutput('npm view "%s" '
'dependencies --json 2>/dev/null' %
module)[1]
dependencies = _parseJSON(npm_out)
else:
dependencies = module.json['dependencies']
module = module.name
except ValueError:
return None
mapper = _Mapper.get_instance()
result = {}
keys = sorted(list(dependencies.keys()))
last_dep = False
for dep in keys:
if dep == keys[-1]:
last_dep = True
result[dep] = {}
result[dep]['version'] = dependencies[dep]
result[dep]['debian'] = mapper.get_debian_package(dep)['repr']
dep_prefix = u'└─' if last_dep else u'├─'
print_formatted_dependency(
u"%s %s (%s)" % (dep_prefix, dep, result[dep]['version']),
result[dep]['debian'], prefix)
if recursive and not dep in expanded_dependencies:
expanded_dependencies.append(dep)
if (result[dep]['debian'] and force) or \
result[dep]['debian'] is None:
new_prefix = "%s%s " % (prefix, u" " if last_dep else u"│ ")
result[dep]['dependencies'] = \
search_for_dependencies(dep, recursive, force, new_prefix,
expanded_dependencies)
else:
result[dep]['dependencies'] = None
return result
def search_for_dependencies(module, recursive=False, force=False,
prefix=u'', expanded_dependencies=[]):
try:
if not isinstance(module, _Npm2Deb):
_debug(1, 'getting dependencies - calling npm view %s' % module)
npm_out = _getstatusoutput('npm view "%s" '
'dependencies --json 2>/dev/null'
% module)[1]
dependencies = _parseJSON(npm_out)
else:
dependencies = module.json['dependencies']
module = module.name
except ValueError:
return None
mapper = _Mapper.get_instance()
result = {}
keys = sorted(list(dependencies.keys()))
last_dep = False
for dep in keys:
if dep == keys[-1]:
last_dep = True
result[dep] = {}
result[dep]['version'] = dependencies[dep]
result[dep]['debian'] = mapper.get_debian_package(dep)['repr']
dep_prefix = u'└─' if last_dep else u'├─'
print_formatted_dependency(u"%s %s (%s)" % (dep_prefix, dep,
result[dep]['version']),
result[dep]['debian'], prefix)
if recursive and not dep in expanded_dependencies:
expanded_dependencies.append(dep)
if (result[dep]['debian'] and force) or \
result[dep]['debian'] is None:
new_prefix = "%s%s " % (prefix, u" " if last_dep else u"│ ")
result[dep]['dependencies'] = \
search_for_dependencies(dep, recursive, force, new_prefix,
expanded_dependencies)
else:
result[dep]['dependencies'] = None
return result
def get_debian_package(self, node_module):
result = {}
result['info'] = None
result['name'] = None
result['version'] = None
result['repr'] = None
if node_module in self.json:
db_package = self.json[node_module]
if 'replace' in db_package:
result['name'] = db_package['replace']
if 'info' in db_package:
result['info'] = ('info', db_package['info'])
self.append_warning('info', node_module, db_package['info'])
elif 'warning' in db_package:
result['info'] = ('warning', db_package['warning'])
self.append_warning('warning', node_module,
db_package['warning'])
elif 'error' in db_package:
result['info'] = ('error', db_package['error'])
self.append_warning('error', node_module, db_package['error'])
else:
result['name'] = 'node-%s' % node_module
if not result['name']:
return result
madison = _getstatusoutput(
'apt-cache madison "%s" | grep Sources' % result['name'])
if madison[0] != 0:
result['name'] = None
return result
tmp = madison[1].split('|')
if len(tmp) >= 2:
result['name'] = tmp[0].strip()
result['version'] = tmp[1].strip()
result['repr'] = '%s (%s)' % (result['name'], result['version'])
return result
def download(self):
utils.debug(1, "downloading %s tarball from npm registry" % self.name)
info = _getstatusoutput('npm pack "%s"' % self.name)
if info[0] is not 0:
exception = "Error downloading package %s\n" % self.name
exception += info[1]
raise ValueError(exception)
tarball_file = info[1].strip('\n')
tarball = tarfile.open(tarball_file)
tarball.extractall()
tarball.close()
# rename extracted directory
_os.rename('package', self.name)
# remove tarball file
_os.remove(tarball_file)
if self.name is not self.debian_name:
utils.debug(2, "renaming %s to %s" % (self.name, self.debian_name))
_os.rename(self.name, self.debian_name)
def get_latest_debian_standards_version():
standards_version = _getstatusoutput(
"rmadison -u debian -s sid debian-policy | cut -d'|' -f2| cut -d'.' -f 1,2,3"
)[1]
return standards_version.strip()
def get_latest_debian_standards_version():
standards_version = _getstatusoutput(
"rmadison -u debian -s sid debian-policy | cut -d'|' -f2| cut -d'.' -f 1,2,3"
)[1]
return standards_version.strip()
def get_npm_version(module_name):
return _getstatusoutput('npm view "%s" version' %
module_name)[1].split('\n')[-2].strip()
def get_npm_version(module_name):
return _getstatusoutput(
'npm view "%s" version' % module_name)[1].split('\n')[-2].strip()
