def enumerate_version(self,
url,
versions_file,
threads=10,
verb='head',
timeout=15):
vf = VersionsFile(versions_file)
hashes = {}
futures = {}
files = vf.files_get()
with ThreadPoolExecutor(max_workers=threads) as executor:
for file_url in files:
futures[file_url] = executor.submit(self.enumerate_file_hash,
url,
file_url=file_url,
timeout=timeout)
for file_url in futures:
hashes[file_url] = futures[file_url].result()
version = vf.version_get(hashes)
# Narrow down using changelog, if accurate.
if vf.has_changelog():
version = self.enumerate_version_changelog(url, version, vf,
timeout)
return version, len(version) == 0
def download_append(self, vg, versions_file, **additional_params):
"""
@param vg an instance of VersionGetterBase, such as SSVersions or
DrupalVersions
@param versions_file the versions_file which corresponds to this
VersionGetterBase, in the filesystem.
@param **aditional_params:
- override_newer: utilize this value instead of calling
newer_get.
"""
versions = VersionsFile(versions_file)
ok = self.confirm('This will download a whole bunch of stuff. OK?')
if ok:
base_folder = mkdtemp() + "/"
# Get information needed.
if 'override_newer' in additional_params:
new = additional_params['override_newer']
else:
majors = versions.highest_version_major(vg.update_majors)
new = vg.newer_get(majors)
if len(new) == 0:
self.error("No new version found, versions.xml is up to date.")
# Get hashes.
dl_files = vg.download(new, base_folder)
extracted_dirs = vg.extract(dl_files, base_folder)
file_sums = vg.sums_get(extracted_dirs, versions.files_get())
versions.update(file_sums)
xml = versions.str_pretty()
# Final sanity checks.
f_temp = NamedTemporaryFile(delete=False)
f_temp.write(xml)
f_temp.close()
call(['diff', '-s', f_temp.name, versions_file])
os.remove(f_temp.name)
ok = self.confirm('Overwrite %s with the new file?' %
versions_file)
if ok:
f_real = open(versions_file, 'w')
f_real.write(xml)
f_real.close()
print "Done."
call(['git', 'status'])
else:
self.error('Aborted.')
else:
self.error('Aborted.')
def test_files_get_all_chlg(self):
changelog_file = 'CHANGELOG.txt'
vf = VersionsFile(self.update_versions_xml)
files = vf.files_get()
files_all = vf.files_get_all()
assert len(files) == len(files_all) - 1
assert changelog_file in files_all
assert not changelog_file in files
def download_append(self, vg, versions_file, **additional_params):
"""
@param vg an instance of VersionGetterBase, such as SSVersions or
DrupalVersions
@param versions_file the versions_file which corresponds to this
VersionGetterBase, in the filesystem.
@param **aditional_params:
- override_newer: utilize this value instead of calling
newer_get.
"""
versions = VersionsFile(versions_file)
ok = self.confirm('This will download a whole bunch of stuff. OK?')
if ok:
base_folder = mkdtemp() + "/"
# Get information needed.
if 'override_newer' in additional_params:
new = additional_params['override_newer']
else:
majors = versions.highest_version_major(vg.update_majors)
new = vg.newer_get(majors)
if len(new) == 0:
self.error("No new version found, versions.xml is up to date.")
# Get hashes.
dl_files = vg.download(new, base_folder)
extracted_dirs = vg.extract(dl_files, base_folder)
file_sums = vg.sums_get(extracted_dirs, versions.files_get())
versions.update(file_sums)
xml = versions.str_pretty()
# Final sanity checks.
f_temp = NamedTemporaryFile(delete=False)
f_temp.write(xml)
f_temp.close()
call(['diff', '-s', f_temp.name, versions_file])
os.remove(f_temp.name)
ok = self.confirm('Overwrite %s with the new file?' %
versions_file)
if ok:
f_real = open(versions_file, 'w')
f_real.write(xml)
f_real.close()
print "Done."
call(['git', 'status'])
else:
self.error('Aborted.')
else:
self.error('Aborted.')
def enumerate_version(self, url, versions_file, threads=10, verb='head',
timeout=15, hide_progressbar=False):
vf = VersionsFile(versions_file)
files = vf.files_get()
changelogs = vf.changelogs_get()
if not hide_progressbar:
p = ProgressBar(sys.stderr, len(files) +
len(changelogs), "version")
hashes = {}
futures = {}
with ThreadPoolExecutor(max_workers=threads) as executor:
for file_url in files:
futures[file_url] = executor.submit(self.enumerate_file_hash,
url, file_url=file_url, timeout=timeout)
for file_url in futures:
if shutdown:
futures[file_url].cancel()
continue
try:
hsh = futures[file_url].result()
hashes[file_url] = hsh
except RuntimeError:
pass
if not hide_progressbar:
p.increment_progress()
version = vf.version_get(hashes)
# Narrow down using changelog, if accurate.
if vf.has_changelog():
version = self.enumerate_version_changelog(url, version, vf, timeout)
if not hide_progressbar:
p.increment_progress()
p.hide()
return version, len(version) == 0
def enumerate_version(self, url, versions_file, threads=10, verb='head', timeout=15):
vf = VersionsFile(versions_file)
hashes = {}
futures = {}
files = vf.files_get()
with ThreadPoolExecutor(max_workers=threads) as executor:
for file_url in files:
futures[file_url] = executor.submit(self.enumerate_file_hash,
url, file_url=file_url, timeout=timeout)
for file_url in futures:
hashes[file_url] = futures[file_url].result()
version = vf.version_get(hashes)
# Narrow down using changelog, if accurate.
if vf.has_changelog():
version = self.enumerate_version_changelog(url, version, vf, timeout)
return version, len(version) == 0