def bump_minor(package):
logging.info("updating %s to %s" %
(package['name'], package['distributions']['stable']))
OWD = os.getcwd()
os.chdir(LOCAL_WORK_COPY)
parser = GemfileParser('Gemfile', 'manageiq')
deps = parser.parse()
for key in deps:
if deps[key]:
for dependency in deps[key]:
if dependency.name == package['name']:
print(dependency.__dict__)
with fileinput.input(files=('Gemfile'),
inplace=True,
backup='.swp') as f:
for line in f:
if '"' + dependency.name + '"' in line:
line = line.replace(
dependency.requirement,
package['distributions']['stable'], 1)
print(line)
os.chdir(OWD)
def update_minor_dependency(package):
"""update_minor_dependency is a trivial approach to update the given package to it's current stable release.
This release will be locked in the Gemfile"""
logging.info("updating %s to %s" %
(package['name'], package['distributions']['stable']))
OWD = os.getcwd()
os.chdir(LOCAL_WORK_COPY)
parser = GemfileParser('Gemfile', 'manageiq')
deps = parser.parse()
# lets loop thru all dependencies and if we found the thing we wanna change
# open the Gemfile, walk thru all lines and change the thing
for key in deps:
if deps[key]:
for dependency in deps[key]:
if dependency.name == package['name']:
with fileinput.input(files=('Gemfile'),
inplace=True,
backup='.swp') as Gemfile:
for line in Gemfile:
if '"' + dependency.name + '"' in line:
line = line.replace(
dependency.requirement,
package['distributions']['stable'], 1)
print(line.replace('\n', '', 1)) # no new line!
os.chdir(OWD)
def test_source_only_gemfile():
gemparser = GemfileParser('tests/Gemfile')
expected = {
'development': [],
'test': [],
'runtime': [],
'metrics': [],
'production': []
}
dependencies = gemparser.parse()
assert_equal(dependencies, expected)
def process(self, path):
'''
Generates dependency list based on provided input file.
**Algorithm**
1. Start
2. Get list of direct dependencies from Gemfile
3. For each dependency in dependency_list, "dep"
a. Find out packaging status of "dep"
b. If "dep" is satisfied in Debian and their dependencies are set
to be ignored, continue to next "dep"
c. Else
i. Get the runtime dependencies of "dep" from Rubygems API
at https://rubygems.org/api/v1/dependencies.json
ii. Add each runtime dependency to dependency_list
'''
self.parser = GemfileParser(path, appname=self.appname)
# self.cached_info = self.get_cache_content()
parsed = self.parser.parse_gemfile(path)
self.original_list = parsed['runtime'] + parsed['production']
self.original_list_name = [x.name for x in self.original_list]
counter = 0
while True:
try:
current_gem = DetailedDependency(self.original_list[counter])
print("Current Gem: %s" % current_gem.name)
if "rails-assets" in current_gem.name:
print("\tRails Assets Found. Skipping")
counter = counter + 1
continue
current_gem.debian_status()
self.dependency_list[current_gem.name] = current_gem
if current_gem.satisfied and self.ignoresatisfied:
print("%s is satisfied in %s" % (current_gem.name,
current_gem.suite))
else:
gem_dependencies = self.get_dependencies(current_gem)
for dep in gem_dependencies:
dep.parent.append(current_gem.name)
if dep.name not in self.original_list_name:
self.original_list.append(dep)
self.original_list_name.append(dep.name)
else:
existing_position = self.original_list_name.\
index(dep.name)
requirement1 = self.original_list[
existing_position].requirement
requirement2 = dep.requirement
stricter_req = get_stricter(requirement1,
requirement2)
if stricter_req != requirement1:
self.original_list[existing_position]\
.requirement = requirement2
counter = counter + 1
pass
except IndexError:
break
def check_gemparser_results(test_file, regen=False):
"""
Run GemfileParser.parse on `test_file` and check against a JSON file that
contains expected results with the same name as the `test_file` with a
"-expected.json" suffix appended.
"""
import json
gemparser = GemfileParser(test_file)
dependencies = {
group: [dep.to_dict() for dep in deps]
for group, deps in gemparser.parse().items()
}
expected_file = test_file + '-expected.json'
if regen:
with open(expected_file, 'w') as o:
json.dump(dependencies, o, indent=2)
with open(expected_file) as o:
expected = json.load(o)
assert expected == dependencies
def test_name():
gemparser = GemfileParser("tests/Gemfile_2")
dependencies = gemparser.parse()
assert_equal(dependencies["runtime"][0].name, "rails")
def parse_file(self, fname, file_content):
parser = GemfileParser(fname)
return parser.parse()
def test_source_only_gemfile():
gemparser = GemfileParser("tests/Gemfile")
expected = {"development": [], "test": [], "runtime": [], "metrics": [], "production": []}
dependencies = gemparser.parse()
assert_equal(dependencies, expected)
def test_gemspec():
gemparser = GemfileParser("tests/Gemfile_2")
dependencies = gemparser.parse()
assert_in("rails", [x.name for x in dependencies["runtime"]])
assert_in("responders", [x.name for x in dependencies["development"]])
def test_source():
gemparser = GemfileParser("tests/Gemfile_2")
dependencies = gemparser.parse()
assert_equal(dependencies["runtime"][0].source, "http://www.example.com")
def test_group_block():
gemparser = GemfileParser("tests/Gemfile_2")
dependencies = gemparser.parse()
assert_equal(dependencies["development"][0].requirement, "3.0.0")
assert_equal(dependencies["runtime"][0].requirement, "4.2.4")
def parse_spec(self, location):
"""
Return dictionary contains podspec or gemspec file data.
"""
with io.open(location, encoding='utf-8', closefd=True) as data:
lines = data.readlines()
spec_data = {}
for line in lines:
line = pre_process(line)
match = self.parse_name.match(line)
if match:
name = match.group('name')
spec_data['name'] = get_stripped_data(name)
match = self.parse_version.match(line)
if match:
version = match.group('version')
spec_data['version'] = get_stripped_data(version)
match = self.parse_license.match(line)
if match:
license_value = match.group('license')
spec_data['license'] = get_stripped_data(license_value)
match = self.parse_summary.match(line)
if match:
summary = match.group('summary')
spec_data['summary'] = get_stripped_data(summary)
match = self.parse_homepage.match(line)
if match:
homepage = match.group('homepage')
spec_data['homepage_url'] = get_stripped_data(homepage)
match = self.parse_source.match(line)
if match:
source = re.sub(r'/*.*source.*?>', '', line)
stripped_source = re.sub(r',.*', '', source)
spec_data['source'] = get_stripped_data(stripped_source)
match = self.parse_description.match(line)
if match:
if location.endswith('.gemspec'):
# FIXME: description can be in single or multi-lines
# There are many different ways to write description.
description = match.group('description')
spec_data['description'] = get_stripped_data(description)
else:
spec_data['description'] = get_description(location)
if '.email' in line:
_key, _sep, value = line.rpartition('=')
stripped_emails = get_stripped_data(value)
stripped_emails = stripped_emails.strip()
stripped_emails = stripped_emails.split(',')
spec_data['email'] = stripped_emails
elif '.author' in line:
authors = re.sub(r'/*.*author.*?=', '', line)
stripped_authors = get_stripped_data(authors)
stripped_authors = re.sub(r'(\s*=>\s*)', '=>',
stripped_authors)
stripped_authors = stripped_authors.strip()
stripped_authors = stripped_authors.split(',')
spec_data['author'] = stripped_authors
parser = GemfileParser(location)
deps = parser.parse()
dependencies = {}
for key in deps:
depends = deps.get(key, []) or []
for dep in depends:
dependencies[dep.name] = dep.requirement
spec_data['dependencies'] = dependencies
return spec_data
def test_gemspec():
gemparser = GemfileParser('tests/Gemfile_2')
dependencies = gemparser.parse()
assert_in('rails', [x.name for x in dependencies['runtime']])
assert_in('responders', [x.name for x in dependencies['development']])
def test_source():
gemparser = GemfileParser('tests/Gemfile_2')
dependencies = gemparser.parse()
assert_equal(dependencies['runtime'][0].source, 'http://www.example.com')
def test_group_block():
gemparser = GemfileParser('tests/Gemfile_2')
dependencies = gemparser.parse()
assert_equal(dependencies['development'][0].requirement, '3.0.0')
assert_equal(dependencies['runtime'][0].requirement, '4.2.4')
def test_group():
gemparser = GemfileParser('tests/Gemfile_3')
dependencies = gemparser.parse()
assert_equal(dependencies['development'][0].requirement, '4.2.4')
def test_requirement():
gemparser = GemfileParser('tests/Gemfile_2')
dependencies = gemparser.parse()
assert_equal(dependencies['runtime'][0].requirement, '4.2.4')
def test_name():
gemparser = GemfileParser('tests/Gemfile_2')
dependencies = gemparser.parse()
assert_equal(dependencies['runtime'][0].name, 'rails')
class GemDeps(object):
'''
Main Class to generate dependency list of a Ruby (on Rails) app.
'''
def __init__(self, appname, ignoresatisfied=True):
'''
Initialize necessary attributes.
'''
self.appname = appname
self.original_list = []
self.dependency_list = {}
self.ignoresatisfied = ignoresatisfied
def process(self, path):
'''
Generates dependency list based on provided input file.
**Algorithm**
1. Start
2. Get list of direct dependencies from Gemfile
3. For each dependency in dependency_list, "dep"
a. Find out packaging status of "dep"
b. If "dep" is satisfied in Debian and their dependencies are set
to be ignored, continue to next "dep"
c. Else
i. Get the runtime dependencies of "dep" from Rubygems API
at https://rubygems.org/api/v1/dependencies.json
ii. Add each runtime dependency to dependency_list
'''
self.parser = GemfileParser(path, appname=self.appname)
# self.cached_info = self.get_cache_content()
parsed = self.parser.parse_gemfile(path)
self.original_list = parsed['runtime'] + parsed['production']
self.original_list_name = [x.name for x in self.original_list]
counter = 0
while True:
try:
current_gem = DetailedDependency(self.original_list[counter])
print("Current Gem: %s" % current_gem.name)
if "rails-assets" in current_gem.name:
print("\tRails Assets Found. Skipping")
counter = counter + 1
continue
current_gem.debian_status()
self.dependency_list[current_gem.name] = current_gem
if current_gem.satisfied and self.ignoresatisfied:
print("%s is satisfied in %s" % (current_gem.name,
current_gem.suite))
else:
gem_dependencies = self.get_dependencies(current_gem)
for dep in gem_dependencies:
dep.parent.append(current_gem.name)
if dep.name not in self.original_list_name:
self.original_list.append(dep)
self.original_list_name.append(dep.name)
else:
existing_position = self.original_list_name.\
index(dep.name)
requirement1 = self.original_list[
existing_position].requirement
requirement2 = dep.requirement
stricter_req = get_stricter(requirement1,
requirement2)
if stricter_req != requirement1:
self.original_list[existing_position]\
.requirement = requirement2
counter = counter + 1
pass
except IndexError:
break
def get_dependencies(self, gem):
'''
Return dependencies of a gem.
'''
print("Getting Dependencies of %s" % gem.name)
api_url = 'https://rubygems.org/api/v1/dependencies.json'
parameters = 'gems=%s' % gem.name
fetch_url = api_url + '?' + parameters
a = urlopen(url=fetch_url)
serialized = json.loads(a.read().decode('utf-8'))
latest_gem = self.smallest_satisfiable(serialized, gem)
dependency_list = []
for dependency in latest_gem['dependencies']:
n = GemfileParser.Dependency()
n.name = dependency[0]
n.requirement = dependency[1].split(',')
dependency_list.append(n)
print n.name, n.requirement
return dependency_list
def smallest_satisfiable(self, serialized, gem):
'''
Get smallest version of gem that satisfies the requirement.
'''
version_list = {}
version_gem_list = []
for gem_version in serialized:
version_list[gem_version['number']] = gem_version
version_gem_list.append(gem_version['number'])
least = least_satisfiable_version(gem.requirement, version_gem_list)
print("Gem name: %s, Requirement: %s, Selected Version: %s" %
(gem.name, gem.requirement, least))
return version_list[least]
def write_output(self, path=None):
'''
Generate output in JSON format to generate statusbar.
'''
if path:
out_path = os.path.join(path, 'debian_status.json')
else:
out_path = 'debian_status.json'
new_list = {}
for dep in self.dependency_list:
new_list[dep] = self.dependency_list[dep].__dict__
with open(out_path, "w") as f:
f.write(json.dumps(new_list, indent=4))
def generate_dot(self, path=None):
'''
Generate output in dot format to generate graphs.
'''
if path:
out_path = os.path.join(path, 'graph.dot')
else:
out_path = 'graph.dot'
dotf = open(out_path, 'w')
dotf.write('digraph %s\n{\n' % self.appname)
for dep in self.dependency_list:
name = dep
color = self.dependency_list[dep].color
block = '"%s"[color=%s];\n' % (name, color)
dotf.write(block)
for parent in self.dependency_list[dep].parent:
dotf.write('"%s"->"%s";\n' %
(parent, self.dependency_list[dep].name))
dotf.write("}")
dotf.close()
def test_requirement():
gemparser = GemfileParser("tests/Gemfile_2")
dependencies = gemparser.parse()
assert_equal(dependencies["runtime"][0].requirement, "4.2.4")
def parse_file(self, fname, file_content):
parser = GemfileParser(fname)
return parser.parse()
def test_group():
gemparser = GemfileParser("tests/Gemfile_3")
dependencies = gemparser.parse()
assert_equal(dependencies["development"][0].requirement, "4.2.4")