def fetchWeatherFiles(self):
hrrrDir = os.path.abspath(self.hrrrDirPath)
fNames = []
# Date when NOAA archive format changed
isReformat = self.runStartUtc > datetime.datetime(
2019, 7, 22, 0, 0, 0, 0, dateutil.tz.tzutc())
for dt in self.computeTimes():
name = dt.strftime('hysplit.%Y%m%d.%Hz.hrrra')
fullPath = hrrrDir + '/' + name
if not os.path.exists(fullPath):
if isReformat:
linkEnd = dt.strftime('%Y%m%d_%H-') + str(
dt.hour + 5).zfill(2) + '_hrrr'
fileDownloaded = download_file(
'https://storage.googleapis.com/high-resolution-rapid-refresh/noaa_arl_formatted/'
+ linkEnd, fullPath)
if not fileDownloaded:
self.log(
'File not found in the Google Cloud Platform HRRR archive. Trying ARL FTP server... '
)
download_file(
'ftp://arlftp.arlhq.noaa.gov/pub/archives/hrrr/' +
linkEnd, fullPath)
else:
download_file(
'ftp://arlftp.arlhq.noaa.gov/pub/archives/hrrr.v1/' +
name, fullPath)
fNames.append(fullPath)
return fNames
def get_update(html_cache=None):
if html_cache:
with open(html_cache) as file:
data_html = file.read()
else:
data_html = requests.get(
"http://wiki.joyme.com/arknights/%E5%B9%B2%E5%91%98%E6%95%B0%E6%8D%AE%E8%A1%A8"
).content.decode()
soup = BeautifulSoup(data_html)
wifis = soup.find("table", id="CardSelectTr").find_all("tr")
ret = []
for i, w in enumerate(wifis):
if i == 0:
continue
info_list = w.find_all("td")
do_man = DoMan(*[re.sub("\s", "", i.text) for i in info_list],
image=info_list[0].div.div.img["src"])
if not os.path.exists(f"../static/image/do_man/{do_man.name}.png"):
download_file(do_man.image,
"../static/image/do_man/",
rename_to=do_man.name)
ret.append(do_man)
return ret
def process(self):
print "Parsing TV series: "
if self.URL_TV_SERIES is None:
print "No url found"
return
with session() as c:
final_results = []
c.post(self.urlLogin, data=self.payload)
#urlTVSeries = 'http://www.cinehub24.com/item/tvseries/english-tv-program/castle-season-7-updated-every-week.aspx'
urlTVSeries = self.URL_TV_SERIES
print "Url",urlTVSeries
response = c.get(urlTVSeries)
html = response.text
links = self.parse_tv_series(html, c)
total_files_for_download = len(links)
k = 1
for link in links:
# If you want to apply range, uncomment this block
#if not (k>=RANGE_LOWER and k<=RANGE_UPPER):
# print "K",k,"Range break"
# k += 1
# continue
print "Downloading", k, "of", total_files_for_download
link = link.encode('utf-8')
#print 'Link', str(link)
utils.download_file(link, self.download_path, BLOCK_SIZE)
k += 1
def load_from_url(self, url, force_reload=False):
"""
Given a URL, this will generate the values
for the Vote from the XML
:param url: The URL where the data can be found -- str
:param force_reload: When True, this will force a refresh of that XML
"""
cache = url.split('://')[-1].replace('/', '_')
self._sources['url'] = url
self._sources['xml'] = self.ROOT_DIR + 'web/' + cache
xml = download_file(self._sources['url'], self._sources['xml'],
force_reload)
soup = BeautifulSoup(xml, 'xml')
try:
self._congress['majority'] = soup.find('majority').text
except AttributeError:
xml = download_file(self._sources['url'], self._sources['xml'],
force_reload)[3:]
soup = BeautifulSoup(xml, 'xml')
self._extract_congressional_info(soup)
self._extract_basic_vote(soup)
self._process_datetime(soup)
self._extract_totals(soup)
self._extract_votes(soup)
self.to_json()
def get_sport(sport):
global root
utils.logmessage("Getting: " + sport, "getdata", 1)
if sport == "baseball":
url = "http://seanlahman.com/files/database/lahman-csv_2014-02-14.zip"
path = root + "rawdata/lahman/"
saveas = "lahman.zip"
if sport == "basketball":
url = "http://www.databasebasketball.com/databasebasketball_2009_v1.zip"
path = root + "rawdata/basketball/"
saveas = "basketball.zip"
# download file
utils.logmessage("Downloading zip file", "getdata", 1)
utils.download_file(url, path+saveas)
# unzip file
utils.unzip(path+saveas, path)
utils.logmessage("Unzipping file", "getdata", 1)
pass
def _prepare_tests(self):
# Sets KUBE_TEST_REPO_LIST
# Builds tests
# Taints linux nodes so that no pods will be scheduled there.
kubectl = utils.get_kubectl_bin()
out, _ = utils.run_shell_cmd([
kubectl, "get", "nodes", "--selector",
"beta.kubernetes.io/os=linux", "--no-headers", "-o",
"custom-columns=NAME:.metadata.name"
])
linux_nodes = out.decode("ascii").strip().split("\n")
for node in linux_nodes:
utils.run_shell_cmd([
kubectl, "taint", "nodes", "--overwrite", node,
"node-role.kubernetes.io/master=:NoSchedule"
])
utils.run_shell_cmd([
kubectl, "label", "nodes", "--overwrite", node,
"node-role.kubernetes.io/master=NoSchedule"
])
self.logging.info("Downloading repo-list")
utils.download_file(self.opts.repo_list, "/tmp/repo-list")
os.environ["KUBE_TEST_REPO_LIST"] = "/tmp/repo-list"
self.logging.info("Building tests")
utils.run_shell_cmd(cmd=["make", 'WHAT="test/e2e/e2e.test"'],
cwd=utils.get_k8s_folder())
self.logging.info("Building ginkgo")
utils.run_shell_cmd(
cmd=["make", 'WHAT="vendor/github.com/onsi/ginkgo/ginkgo"'],
cwd=utils.get_k8s_folder())
self._setup_kubetest()
def install_loot_api(version, revision, dl_dir, destination_path):
url = ("https://github.com/loot/loot-api-python/releases/download/"
"{0}/loot_api_python-{0}-0-g{1}_master-python2.7-win32.7z".format(
version, revision))
archive_path = os.path.join(dl_dir, "loot_api.7z")
seven_zip_folder = os.path.join(MOPY_PATH, "bash", "compiled")
seven_zip_path = os.path.join(seven_zip_folder, "7z.exe")
loot_dll = os.path.join(destination_path, "loot.dll")
loot_api_pyd = os.path.join(destination_path, "loot_api.pyd")
if os.path.exists(loot_dll):
os.remove(loot_dll)
if os.path.exists(loot_api_pyd):
os.remove(loot_api_pyd)
LOGGER.info("Downloading LOOT API Python wrapper...")
LOGGER.debug("Download url: {}".format(url))
LOGGER.debug(
"Downloading LOOT API Python wrapper to {}".format(archive_path))
utils.download_file(url, archive_path)
LOGGER.info("Extracting LOOT API Python wrapper to " +
utils.relpath(destination_path))
command = [
seven_zip_path,
"e",
archive_path,
"-y",
"-o" + destination_path,
"*/loot.dll",
"*/loot_api.pyd",
]
utils.run_subprocess(command, LOGGER)
os.remove(archive_path)
def test_evaluation_electroatomic():
download_file(
'http://t2.lanl.gov/nis/data/data/ENDFB-VII-electroatomic/Mo/nat',
'Mo.txt', '2139a23258c517ae3bfa5f2cc346da4c')
mo = Evaluation('Mo.txt', verbose=False)
mo.read()
assert mo.info['laboratory'].startswith('LLNL')
assert 'Cullen' in mo.info['author']
assert mo.info['library'] == (u'ENDF/B', 6, 0)
assert mo.info['sublibrary'] == 113
assert 526 in mo.reactions
assert 527 in mo.reactions
assert 543 in mo.reactions
# Check bremsstrahlung cross section
brem = mo.reactions[527]
E = np.logspace(1, 11, 10)
xs = np.array([
4276.9, 6329.94427327, 5350.43318579, 1818.71320602, 467.65208672,
376.17541997, 434.15352828, 484.99421137, 535.39666445, 591.138
])
assert_array_almost_equal(brem.xs(E), xs)
# Check bremsstrahlung secondary distributions
assert brem.products[0]['za'] == 11 # electrons
E = np.logspace(1, 11)
assert np.all(brem.products[0]['yield'](E) == 1.0)
eout = np.array([
0.1, 0.133352, 0.165482, 0.228757, 0.316228, 0.421697, 0.523299,
0.723394, 1., 1.41421, 2., 2.82843, 4., 5.65685, 8., 9.9, 10.
])
assert_array_almost_equal(brem.products[0]['energy_out'][0], eout)
def test_evaluation_decay():
download_file('http://t2.lanl.gov/nis/data/endf/decayVII.1/092_U_233',
'U233.txt', '3db23dc650bae28eabb92942dd7d0de5')
u233 = Evaluation('U233.txt', verbose=False)
u233.read()
assert hasattr(u233, 'info')
assert u233.info['library'] == (u'ENDF/B', 7, 1)
assert u233.info['sublibrary'] == 4
assert u233.material == 3513
assert u233.target['mass'] == 231.0377
assert u233.target['zsymam'] == u' 92-U -233 '
assert not u233.target['stable']
assert u233.decay['half_life'] == (5023970000000.0, 6311520000.0)
assert u233.decay['energies'] == [(5043.237, 536.3191),
(1110.218, 107.6781),
(4888351.0, 28967.6)]
assert len(u233.decay['modes']) == 1
assert u233.decay['modes'][0]['branching_ratio'] == (1.0, 0.0)
assert u233.decay['modes'][0]['energy'] == (4908500.0, 1200.0)
assert u233.decay['modes'][0]['type'] == u'alpha'
for s in ['e-', 'alpha', 'xray', 'gamma']:
assert s in u233.decay['spectra']
assert u233.decay['spectra']['e-']['energy_average'] == (5043.237,
536.3191)
alpha_spectrum = u233.decay['spectra']['alpha']['discrete']
assert alpha_spectrum[-1]['energy'] == (4824200.0, 1200.0)
assert alpha_spectrum[-1]['intensity'] == (0.843, 0.006)
def test_u235():
download_file("http://t2.lanl.gov/nis/data/data/ENDFB-VII.1-neutron/U/235",
"U235.txt", "1b71da3769d8b1e675c3c579ba5cb2d3")
u235 = Library('U235.txt')
nuc = 922350000
u235._read_res(nuc)
u235._read_xs(nuc, 37)
exp_a = array_from_ENDF(
io.BytesIO(
b""" 9.223500+4 2.330248+2 0 0 0 0
-1.788560+7-1.788560+7 0 0 1 6
6 2 """ + b"""
1.796240+7 5.05980-10 1.800000+7 3.810030-7 1.850000+7 8.441785-5
1.900000+7 2.387410-4 1.950000+7 1.348763-3 2.000000+7 4.785594-3
"""))
obs = u235.structure[nuc]['data'][nuc]['xs'][37][0]
exp = {
'intpoints': 6,
'intschemes': 2,
'e_int': exp_a[3:5].flat[::2],
'xs': exp_a[3:5].flat[1::2]
}
for key in obs:
assert_array_equal(obs[key], exp[key])
def test_loadfile():
download_file(
"https://www-nds.iaea.org/epdl97/data/endl/eedl/za082000",
"epdl97_eedl_Pb",
"502105669e0c0ad917301d219c649aaf",
)
testlib = Library("epdl97_eedl_Pb")
# test the nuclides
pb_nuclide = 820000000
exp_nuclides = [pb_nuclide]
obs_nuclides = list(map(int, testlib.structure.keys()))
assert_array_equal(exp_nuclides, obs_nuclides)
# test the incoming particles
exp_pin = [9]
obs_pin = testlib.structure[pb_nuclide]["pin"]
assert_array_equal(sorted(exp_pin), sorted(obs_pin))
# test the reaction properties
exp_rdesc = [7, 8, 10, 81, 82, 83]
obs_rdesc = testlib.structure[pb_nuclide]["rdesc"]
assert_array_equal(sorted(exp_rdesc), sorted(obs_rdesc))
# test the reaction descriptors
exp_rprop = [0, 10, 11, 21, 22]
obs_rprop = testlib.structure[pb_nuclide]["rprop"]
assert_array_equal(sorted(exp_rprop), sorted(obs_rprop))
def ieee_index_page(punumber):
"""Return a list [title] and [link] for each conference year. The
year,link pair should be determined outside.
"""
# if not exist, download to a tmp file
# https://ieeexplore.ieee.org/xpl/conhome.jsp?punumber=1000639
# parse the file
# get the list of punumber for each year, and return
url = 'https://ieeexplore.ieee.org/xpl/conhome.jsp?punumber=' + str(
punumber)
html_file = download_to_hash(url)
if not os.path.exists(html_file):
download_file(url, html_file)
soup = BeautifulSoup(open(html_file), 'lxml')
texts = []
links = []
for a in soup.select('.detail li a'):
text = a.get_text().strip()
link = a['href']
if not link.startswith('http'):
link = 'https://ieeexplore.ieee.org/xpl/' + link
texts.append(text)
links.append(link)
return list(reversed(texts)), list(reversed(links))
def main():
# 1 初始化界面
# Render the readme as markdown using st.markdown.
readme_text = st.markdown(read_markdown("instructions_yolov3.md"))
# 2 下载yolov3的模型文件
# Download external dependencies.
for filename in EXTERNAL_DEPENDENCIES.keys():
download_file(filename)
# 下载yolov3文件
# Once we have the dependencies, add a selector for the app mode on the sidebar.
st.sidebar.title("图像检测参数调节器") # 侧边栏
app_mode = st.sidebar.selectbox(
"切换页面模式:",
["Run the app", "Show instructions", "Show the source code"])
# 展示栏目三
if app_mode == "Run the app":
#readme_text.empty() # 刷新页面
st.markdown('---')
st.markdown('## YOLOv3 检测结果:')
run_the_app() # 运行内容
# 展示栏目一
elif app_mode == "Show instructions":
st.sidebar.success('To continue select "Run the app".')
# 展示栏目二
elif app_mode == "Show the source code":
readme_text.empty() # 刷新页面
st.code(read_markdown("streamlit_app_yolov3.py"))
def data_generator(name):
files = get_files()
local_path = get_data_absolute_path(files[name]['local_path'])
download_file(url=files['name']['url'], dest_path=local_path)
with gzip.open(local_path) as in_file:
for line in in_file:
yield line
def download_redists():
"""Downloads all required MSVC redistributables if they're not already
present."""
if not os.path.isdir(REDIST_PATH):
os.makedirs(REDIST_PATH)
msvc_2010_x86 = os.path.join(REDIST_PATH, u'vcredist_2010_x86.exe')
if not os.path.isfile(msvc_2010_x86):
LOGGER.info(u'MSVC 2010 x86 redistributable not found, downloading')
utils.download_file(
u'https://download.microsoft.com/download/5/B/C/'
u'5BC5DBB3-652D-4DCE-B14A-475AB85EEF6E/'
u'vcredist_x86.exe', msvc_2010_x86)
LOGGER.debug(u'MSVC 2010 x86 redistributable downloaded successfully')
else:
LOGGER.debug(u'MSVC 2010 x86 redistributable found')
msvc_2010_x64 = os.path.join(REDIST_PATH, u'vcredist_2010_x64.exe')
if not os.path.isfile(msvc_2010_x64):
LOGGER.info(u'MSVC 2010 x64 redistributable not found, downloading')
utils.download_file(
u'https://download.microsoft.com/download/3/2/2/'
u'3224B87F-CFA0-4E70-BDA3-3DE650EFEBA5/'
u'vcredist_x64.exe', msvc_2010_x64)
LOGGER.debug(u'MSVC 2010 x64 redistributable downloaded successfully')
else:
LOGGER.debug(u'MSVC 2010 x64 redistributable found')
def get_chromagram(self, mode=MusicDataType.AUDIO):
"""
Get chromagram for the spotify track
Creates chromagram either from mp3(:meth:`utils.get_chromagram_from_audio`) or echonest analysis (:meth:`apiwrappers.echonest.get_echonestsong_by_spotifyid`) of the spotify track.Chromagram may return None if there is no matching echonest song for the spotify track
Args:
mode: :class:`.MusicDataType`. either should be audio or echonest
Returns:
Chromagram representation.
"""
directory = os.path.dirname(os.path.abspath(__file__))
if mode == MusicDataType.AUDIO:
audio_file_path = directory + '/' + self.id + '.mp3'
utils.download_file(self.preview_url, audio_file_path)
chromagram = utils.get_chromagram_from_audio(audio_file_path)
utils.remove_file(audio_file_path)
return chromagram
elif mode == MusicDataType.ECHONEST:
e = get_echonestsong_by_spotifyid(self.id)
if e:
self.echonest_features = e.features
return e.features.chroma
else:
print "No echonest song found for spotify:track:"+self.id
return None
def get_nsis_root(cmd_arg):
""" Finds and returns the nsis root folder. """
if cmd_arg is not None:
LOGGER.debug("User provided NSIS path at {}".format(cmd_arg))
return cmd_arg
try:
nsis_path = winreg.QueryValue(winreg.HKEY_LOCAL_MACHINE,
r"Software\NSIS")
LOGGER.debug("Found system NSIS path at {}".format(nsis_path))
return nsis_path
except WindowsError:
pass
if not os.path.isdir(NSIS_PATH):
LOGGER.debug("Local NSIS not found at {}".format(NSIS_PATH))
local_build_path = os.path.dirname(NSIS_PATH)
nsis_url = ("https://sourceforge.net/projects/nsis/files/"
"NSIS%203/{0}/nsis-{0}.zip/download".format(NSIS_VERSION))
dl_dir = tempfile.mkdtemp()
nsis_zip = os.path.join(dl_dir, "nsis.zip")
LOGGER.info("Downloading NSIS {}...".format(NSIS_VERSION))
LOGGER.debug("Download url: {}".format(nsis_url))
LOGGER.debug("Download NSIS to {}".format(nsis_zip))
utils.download_file(nsis_url, nsis_zip)
with zipfile.ZipFile(nsis_zip) as fzip:
fzip.extractall(local_build_path)
os.remove(nsis_zip)
os.rename(
os.path.join(local_build_path, "nsis-{}".format(NSIS_VERSION)),
NSIS_PATH,
)
return NSIS_PATH
def download_extras(self, base_path):
"""
"""
# https://github.com/niqdev/packtpub-crawler/pull/27
if self.__config.has_option('path', 'path.group'):
folder_name = self.info['title'].encode('ascii', 'ignore').replace(' ', '_') + \
self.info['author'].encode('ascii', 'ignore').replace(' ', '_')
directory = base_path + join(
self.__config.get('path', 'path.ebooks'), folder_name,
self.__config.get('path', 'path.extras'))
else:
directory = base_path + self.__config.get('path', 'path.extras')
url_image = self.info['url_image']
filename = self.info['filename'] + '_' + split(url_image)[1]
self.info['paths'].append(
download_file(self.__session, url_image, directory, filename,
self.__headers))
if 'url_source_code' in self.info:
self.info['paths'].append(
download_file(self.__session, self.info['url_source_code'],
directory, self.info['filename'] + '.zip',
self.__headers))
def download_all(self, directory):
"""Download all the target_links into a directory.
Args:
directory (str): Path to a directory to download files.
"""
# start timer
start = time.time()
for index, link in enumerate(self.target_links):
download_link = link[1]
filename = download_link.split("/")[-1]
path = os.path.join(directory, filename)
# don't download the same file twice
if not os.path.isfile(path):
utils.download_file(download_link, filename, directory)
# print a statement with the progress of the download
status = " ".join(["Downloaded", str(index + 1), "of",
str(len(self.target_links)), "files."])
print(status, end="\r" * len(status))
# calculate how many minutes took
elasped = time.time() - start
print("Finished in", elasped / 60.0, "minutes")
def task_verify(wav_url, person_id):
start_time = time.time()
print('开始时间：',
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(start_time)))
m = ModelInterface.load(model)
if person_id not in m.features:
return 'fail', 'current user not trained', ''
# 下载训练语音文件，
current_time = time.strftime("%Y%m%d%H%I%S", time.localtime(time.time()))
dest_wav = verify_voice_dir + current_time + '_' + person_id + '.wav'
utils.download_file(wav_url, dest_wav)
for f in glob.glob(os.path.expanduser(dest_wav)):
fs, signal = utils.read_wav(f)
probability = m.verify(fs, signal, person_id)
print(probability)
if probability > -48:
print(f, '-> 匹配成功 ：', person_id)
return 'success', '', 'yes'
else:
print(f, '->未匹配成功')
return 'success', '', 'no'
end_time = time.time()
print('结束时间：', time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(end_time)))
print('共耗时', end_time - start_time)
def scrape(self, url):
try:
# get image
print "Retrieving largest image of %s... " % url
dest_dir = os.path.dirname(
os.getcwd()) + '/scraped_imgs/' + urlparse(
url).netloc + '/' + time.strftime('%Y-%m-%d %H:%M:%S')
if os.name == "nt":
dest_dir = os.path.dirname(
os.getcwd()) + '\\scraped_imgs\\' + urlparse(
url).netloc + '\\' + time.strftime('%Y-%m-%d %H-%M-%S')
print dest_dir
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
html = requests.get(url).text
soup = BeautifulSoup(html, "html.parser")
file_urls = get_file_urls(url, soup)
largest_file_url, size = get_largest_file_url(file_urls)
print "Downloading largest image file: " + largest_file_url
download_file(largest_file_url, dest_dir)
file_summary = 'Largest Image File: ' + str(
largest_file_url) + ' Sized: ' + str(size)
status = "Success"
except Exception, e:
file_summary = "Scrape Failed. Reason: %s" % str(e)
status = "Failure"
def install_webui():
nodejs_source_url = ctx.node.properties['nodejs_tar_source_url']
webui_source_url = ctx.node.properties['webui_tar_source_url']
grafana_source_url = ctx.node.properties['grafana_tar_source_url']
# injected as an input to the script
ctx.instance.runtime_properties['influxdb_endpoint_ip'] = \
os.environ.get('INFLUXDB_ENDPOINT_IP')
nodejs_home = '/opt/nodejs'
webui_home = '/opt/cloudify-ui'
webui_log_path = '/var/log/cloudify/webui'
grafana_home = '{0}/grafana'.format(webui_home)
webui_user = '******'
webui_group = 'webui'
ctx.logger.info('Installing Cloudify\'s WebUI...')
utils.set_selinux_permissive()
utils.copy_notice('webui')
utils.mkdir(nodejs_home)
utils.mkdir(webui_home)
utils.mkdir('{0}/backend'.format(webui_home))
utils.mkdir(webui_log_path)
utils.mkdir(grafana_home)
utils.create_service_user(webui_user, webui_home)
ctx.logger.info('Installing NodeJS...')
nodejs = utils.download_file(nodejs_source_url)
utils.untar(nodejs, nodejs_home)
ctx.logger.info('Installing Cloudify\'s WebUI...')
webui = utils.download_file(webui_source_url)
utils.untar(webui, webui_home)
ctx.logger.info('Installing Grafana...')
grafana = utils.download_file(grafana_source_url)
utils.untar(grafana, grafana_home)
ctx.logger.info('Deploying WebUI Configuration...')
utils.deploy_blueprint_resource(
'{0}/gsPresets.json'.format(CONFIG_PATH),
'{0}/backend/gsPresets.json'.format(webui_home))
ctx.logger.info('Deploying Grafana Configuration...')
utils.deploy_blueprint_resource(
'{0}/grafana_config.js'.format(CONFIG_PATH),
'{0}/config.js'.format(grafana_home))
ctx.logger.info('Fixing permissions...')
utils.chown(webui_user, webui_group, webui_home)
utils.chown(webui_user, webui_group, nodejs_home)
utils.chown(webui_user, webui_group, webui_log_path)
utils.logrotate('webui')
utils.systemd.configure('webui')
def install_webui():
nodejs_source_url = ctx.node.properties['nodejs_tar_source_url']
webui_source_url = ctx.node.properties['webui_tar_source_url']
grafana_source_url = ctx.node.properties['grafana_tar_source_url']
# injected as an input to the script
ctx.instance.runtime_properties['influxdb_endpoint_ip'] = \
os.environ.get('INFLUXDB_ENDPOINT_IP')
nodejs_home = '/opt/nodejs'
webui_home = '/opt/cloudify-ui'
webui_log_path = '/var/log/cloudify/webui'
grafana_home = '{0}/grafana'.format(webui_home)
webui_user = '******'
webui_group = 'webui'
ctx.logger.info('Installing Cloudify\'s WebUI...')
utils.set_selinux_permissive()
utils.copy_notice('webui')
utils.mkdir(nodejs_home)
utils.mkdir(webui_home)
utils.mkdir('{0}/backend'.format(webui_home))
utils.mkdir(webui_log_path)
utils.mkdir(grafana_home)
utils.create_service_user(webui_user, webui_home)
ctx.logger.info('Installing NodeJS...')
nodejs = utils.download_file(nodejs_source_url)
utils.untar(nodejs, nodejs_home)
ctx.logger.info('Installing Cloudify\'s WebUI...')
webui = utils.download_file(webui_source_url)
utils.untar(webui, webui_home)
ctx.logger.info('Installing Grafana...')
grafana = utils.download_file(grafana_source_url)
utils.untar(grafana, grafana_home)
ctx.logger.info('Deploying WebUI Configuration...')
utils.deploy_blueprint_resource(
'{0}/gsPresets.json'.format(CONFIG_PATH),
'{0}/backend/gsPresets.json'.format(webui_home))
ctx.logger.info('Deploying Grafana Configuration...')
utils.deploy_blueprint_resource(
'{0}/grafana_config.js'.format(CONFIG_PATH),
'{0}/config.js'.format(grafana_home))
ctx.logger.info('Fixing permissions...')
utils.chown(webui_user, webui_group, webui_home)
utils.chown(webui_user, webui_group, nodejs_home)
utils.chown(webui_user, webui_group, webui_log_path)
utils.logrotate('webui')
utils.systemd.configure('webui')
def test_loadtape():
download_file("http://www.nndc.bnl.gov/endf/b6.8/tapes/tape.100",
"endftape.100", "b56dd0aee38bd006c58181e473232776")
testlib = Library("endftape.100")
exp_nuclides = set(
[10010000, 30060000, 40090000, 50110000, 30070000, 60000000, 50100000])
obs_nuclides = set(map(int, testlib.structure.keys()))
assert_equal(exp_nuclides, obs_nuclides)
def download_resnet50_weights():
if not os.path.exists(RESNET50_WEIGHTS_FILE):
download_file(url=RESNET50_WEIGHTS_UR,
filepath=RESNET50_WEIGHTS_FILE,
description='Downloading weights')
print("File with ResNet-50 weights: {}".format(RESNET50_WEIGHTS_FILE))
return RESNET50_WEIGHTS_FILE
def install_examples():
logging.debug('Trying to download and unzip ' + EXAMPLES_ZIP)
download_file(EXAMPLES_URL, EXAMPLES_ZIP)
logging.debug(EXAMPLES_ZIP + ' was downloaded')
with zipfile.ZipFile(EXAMPLES_ZIP, 'r') as zip_ref:
zip_ref.extractall(EXAMPLES_DIR)
logging.debug(EXAMPLES_ZIP + ' was unzipped')
logging.debug(EXAMPLES_ZIP + ' is OK')
def download_masterlist(repository, version, dl_path):
url = "https://raw.githubusercontent.com/loot/{}/v{}/masterlist.yaml".format(
repository, version
)
LOGGER.info("Downloading {} masterlist...".format(repository))
LOGGER.debug("Download url: {}".format(url))
LOGGER.debug("Downloading {} masterlist to {}".format(repository, dl_path))
utils.download_file(url, dl_path)
def get_addon(source):
dest = tempfile.mkdtemp()
id = source.split('/')[-1]
if source.endswith('/'):
id = source.split('/')[-2]
destfile = os.path.join(dest, id)
download_file(destfile, source)
return destfile
def download_files(year: int) -> None:
path = utils.download_file(YEAR_URL.format(year=year),
'gazette/{}.html'.format(year))
text = path.read_text()
for m in re.finditer(r'text/(\d+_\w+\d+)_Registrar.cfm', text):
docid = m.group(1)
utils.download_file(DOC_URL.format(year=year, docid=docid),
'gazette/{}/{}.txt'.format(year, docid))
def get_extension(id, url, _type, download=True):
id = id.strip()
if not id:
return
id = id.split('/')[-1]
if (check_manifest_exists and os.path.exists(
os.path.join(root, _type + '-manifests', id + '.json'))):
print('Manifest %s already exists, skipping' % id)
return
if id in badURLs:
return
curdir = os.getcwd()
dest = tempfile.mkdtemp(dir=myTmp)
os.chmod(dest, stat.S_IWRITE)
os.chdir(dest)
destfile = os.path.join(dest, id + '.zip')
print('Downloading...', id)
try:
download_file(destfile, url)
try:
if unzip:
unzip_file(destfile)
except:
print(url)
print('...unzip failed')
badURLs.add(id)
os.chdir(curdir)
return dest
except (UnicodeDecodeError, requests.exceptions.HTTPError) as exc:
# print(exc.message)
print(url)
print('...failed')
badURLs.add(id)
os.chdir(curdir)
return dest
if copy_manifest:
manifest = os.path.join(dest, 'manifest.json')
try:
shutil.copy('manifest.json',
os.path.join(root, _type + '-manifests', id + '.json'))
print('Got manifest for', id)
except:
print('No manifest found for', id)
json_file = os.path.join(root, _type + '-apis', id + '.json')
if parse:
res = examine(dest)
json.dump(res, open(json_file, 'w'))
os.chdir(curdir)
return dest
def get_all_chapters():
r = requests.get(INDEX_ENDPOINT%SUBJECT)
soup = BeautifulSoup(r.text)
links = soup.find_all("a",{"target":"_top"})
os.makedirs(SUBJECT)
for link in links:
if(re.match(r'^/'+SUBJECT,link['href'])):
filename = link['href'].split('/')[-1]
download_file(DOWNLOAD_ENDPOINT%(SUBJECT,filename.split('.')[0]),SUBJECT+'/'+filename.split('.')[0])
def get_chrome_addon(source):
dest = tempfile.mkdtemp()
id = source.split('/')[-1]
if source.endswith('/'):
id = source.split('/')[-2]
url = 'https://clients2.google.com/service/update2/crx?response=redirect&prodversion=46.0&x=id%3D' + id + '%26uc'
destfile = os.path.join(dest, id + '.zip')
download_file(destfile, url)
return destfile
def main():
resolution = args.resolution
assert torch.cuda.is_available()
torch.backends.cudnn.benchmark = True
model_fname = 'deeplab_model/deeplab_model.pth'
dataset_root = 'ffhq_aging{}x{}'.format(resolution, resolution)
assert os.path.isdir(dataset_root)
dataset = CelebASegmentation(dataset_root, crop_size=513)
if not os.path.isfile(resnet_file_spec['file_path']):
print('Downloading backbone Resnet Model parameters')
with requests.Session() as session:
download_file(session, resnet_file_spec)
print('Done!')
model = getattr(deeplab, 'resnet101')(pretrained=True,
num_classes=len(dataset.CLASSES),
num_groups=32,
weight_std=True,
beta=False)
model = model.cuda()
model.eval()
if not os.path.isfile(deeplab_file_spec['file_path']):
print('Downloading DeeplabV3 Model parameters')
with requests.Session() as session:
download_file(session, deeplab_file_spec)
print('Done!')
checkpoint = torch.load(model_fname)
state_dict = {
k[7:]: v
for k, v in checkpoint['state_dict'].items() if 'tracked' not in k
}
model.load_state_dict(state_dict)
for i in range(len(dataset)):
inputs = dataset[i]
inputs = inputs.cuda()
outputs = model(inputs.unsqueeze(0))
_, pred = torch.max(outputs, 1)
pred = pred.data.cpu().numpy().squeeze().astype(np.uint8)
imname = os.path.basename(dataset.images[i])
mask_pred = Image.fromarray(pred)
mask_pred = mask_pred.resize((resolution, resolution), Image.NEAREST)
try:
mask_pred.save(dataset.images[i].replace(
imname, 'parsings/' + imname[:-4] + '.png'))
except FileNotFoundError:
os.makedirs(
os.path.join(os.path.dirname(dataset.images[i]), 'parsings'))
mask_pred.save(dataset.images[i].replace(
imname, 'parsings/' + imname[:-4] + '.png'))
print('processed {0}/{1} images'.format(i + 1, len(dataset)))
def download_extras_dir(self,directory):
"""
"""
url_image = self.info['url_image']
filename = self.info['filename'] + '_' + split(url_image)[1]
self.info['paths'].append(download_file(self.__session, url_image, directory, filename))
if 'url_source_code' in self.info:
self.info['paths'].append(download_file(self.__session, self.info['url_source_code'], directory,
self.info['filename'] + '.zip'))
def get_content(driver, chpt_url, out_html_name, out_audio_name):
utils.open_url(driver, chpt_url, verbose=True)
elem = driver.find_element_by_xpath('//div[@id="bible_chapter_content"]')
save_html(elem.get_attribute('outerHTML'), out_html_name)
try:
audio_url = driver.find_element_by_xpath('//audio').get_attribute('src')
utils.download_file(audio_url, out_audio_name)
except:
pass
def populate_class_labels():
class_file_name = 'yolov3_classes.txt'
class_file_abs_path = dest_dir + os.path.sep + class_file_name
url = 'https://github.com/arunponnusamy/object-detection-opencv/raw/master/yolov3.txt'
if not os.path.exists(class_file_abs_path):
download_file(url=url, file_name=class_file_name, dest_dir=dest_dir)
f = open(class_file_abs_path, 'r')
classes = [line.strip() for line in f.readlines()]
return classes
def download_extras(self):
"""
"""
directory = self.__config.get('path', 'path.extras')
url_image = self.info['url_image']
filename = self.info['filename'] + '_' + split(url_image)[1]
self.info['paths'].append(download_file(self.__session, url_image, directory, filename))
if 'url_source_code' in self.info:
self.info['paths'].append(download_file(self.__session, self.info['url_source_code'], directory,
self.info['filename'] + '.zip'))
def download_tbb_tarball(tbb_ver, dl_dir=""):
tbb_url = get_url_by_tbb_ver(tbb_ver)
base_dir = dl_dir if dl_dir else cm.TBB_BASE_DIR
tarball_path = os.path.join(base_dir, get_tbb_filename(tbb_ver))
if not os.path.isfile(tarball_path):
wl_log.info("Will download %s to %s" % (tbb_url, tarball_path))
ut.download_file(tbb_url, tarball_path)
ut.extract_tbb_tarball(tarball_path)
if verify_tbb_tarball(tbb_ver, tarball_path, tbb_url):
return tarball_path
# we cannot verify the integrity of the downloaded tarball
raise cm.TBBTarballVerificationError("Cannot verify the integrity of %s"
% tarball_path)
def build_iso():
usage = "usage: %prog [options]"
parser = OptionParser(usage)
parser.add_option("--input-iso",
help="Path or URL to ISO file. Default is 'input.iso'.",
action="store", type="string", dest="input_iso",
default='input.iso')
parser.add_option("--output-iso",
help="Path to ISO to generate. Default is 'output.iso'.",
action="store", type="string", dest="output_iso",
default='output.iso')
parser.add_option("--preseed", help="Path or URL to preseed file. " \
"Default is 'preseed.cfg'.", action="store",
type="string", dest="preseed_file",
default='preseed.cfg')
parser.add_option("--hide-boot-loader", help="Hide boot loader (default).",
action='store_true', dest="is_boot_loader_hidden",
default=True)
parser.add_option("--show-boot-loader", help="Show boot loader.",
action='store_false', dest="is_boot_loader_hidden")
(options, args) = parser.parse_args()
tmp_input_iso = None
tmp_preseed_file = None
try:
# Download ISO file if necessary.
if is_url(options.input_iso):
file_handle, tmp_input_iso = tempfile.mkstemp()
download_iso_file(options.input_iso, tmp_input_iso)
options.input_iso = tmp_input_iso
# Download preseed file if necessary.
if is_url(options.preseed_file):
file_handle, tmp_preseed_file = tempfile.mkstemp()
download_file(options.preseed_file, tmp_preseed_file)
options.preseed_file = tmp_preseed_file
# Check that input files exist.
if not os.path.exists(options.preseed_file):
parser.error('No such preseed file %s' % options.preseed_file)
if not os.path.exists(options.input_iso):
parser.error('No such input ISO %s' % options.input_iso)
# Build ISO!
insert_preseed_into_iso(options.preseed_file, options.input_iso,
options.output_iso,
options.is_boot_loader_hidden)
if os.path.exists(options.output_iso):
print "SUCCESS: %s file has been generated." % options.output_iso
finally:
if tmp_input_iso:
os.unlink(tmp_input_iso)
if tmp_preseed_file:
os.unlink(tmp_preseed_file)
def get_chromagram(self,mode=MusicDataType.AUDIO):
directory = os.path.dirname(os.path.abspath(__file__))
if mode == MusicDataType.AUDIO:
audio_file_path = directory + '/' + self.id + '.mp3'
utils.download_file(self.preview_url, audio_file_path)
chromagram = utils.get_chromagram_from_audio(audio_file_path)
utils.remove_file(audio_file_path)
return chromagram
elif mode == MusicDataType.ECHONEST:
if self.features:
return self.features.chroma
else:
print "No echonest song found for:"+self.id
return None
def install_packages(config, **_):
""" Installs a list of packages """
if config is None:
config = ctx.node.properties['config']
ctx.logger.info('Attempting to install packages')
distro = platform.linux_distribution(full_distribution_name=False)
distro_lower = [x.lower() for x in distro]
ctx.logger.info('Working environment: {0} {2} v{1}'.format(distro[0], distro[1], distro[2]))
ctx.logger.info(config)
if 'custom_repo' in config:
_add_custom_repo(config['custom_repo'], distro_lower)
package_list = config['package_list']
for package_to_install in package_list:
# Install from repository (yum / apt)
if 'http' not in package_to_install:
ctx.logger.info('Installing from repository: {0}'.format(package_to_install))
install_command = _get_install_command(
distro=distro_lower,
install_from_repo=True,
package=package_to_install)
# TODO: apt-get update should not get called every install
run(APT_GET_UPDATE)
# Install from package
else:
ctx.logger.info('Installing from URL: {0}'.format(package_to_install))
package = tempfile.mkstemp()
package_path = package[1]
download_file(source=package_to_install, destination=package_path)
install_command = _get_install_command(
distro=distro_lower,
install_from_repo=False,
package=package_path)
ctx.logger.info('Running command: {0}'.format(install_command))
run(install_command)
def execute_before_bootstrap():
exec_paths = ctx_properties['execute_before_bootstrap']
for path in exec_paths:
# TODO: Upon moving to Python 3, convert to urllib2.urlparse
if '://' in path and path.split('://', 1)[0] in ('http', 'https'):
path = utils.download_file(path)
utils.chmod('744', path)
utils.run(path)
def dump_all_library(self):
# self.__GET_login()
# wait(self.__delay)
# self.__POST_login()
# wait(self.__delay)
url = self.__url_base + self.__config.get('url', 'url.myebooks')
response = self.__session.get(url, headers=self.__headers)
self.__log_response(response)
soup = make_soup(response)
for a in soup.findAll('div', attrs={'class': 'product-line unseen'}):
log_info("[+] Downloading : " + a.attrs.get('title'))
#print "Downloading : " + a.attrs.get('title')
directory = a.attrs.get('title')[:-8].replace(' ', '_')
directory = directory.encode('ascii', 'ignore').replace('/', '-') ##### Error - UnicodeEncodeError: 'ascii' codec can't encode character u'\u2019' in position
filename = directory
#print "Directory: " + a.attrs.get('title')[:-8].replace(' ', '_')
# print a
# print "Cover URL: " "http:" + a.find('img', attrs={'class': ' imagecache imagecache-thumbview'}).get('src').replace('thumbview', 'dotd_main_image')
cover_url = a.find('img', attrs={'class': ' imagecache imagecache-thumbview'}).get('src').replace('thumbview', 'dotd_main_image')
download_file(self.__session, 'http:' + cover_url, self.__config.get('path', 'path.dumps') + '/' + directory, filename +'.jpg')
links = []
for link in a.findAll('a', href=True):
url = link.attrs.get('href')
if not '#' in url:
links.append(url)
for i in range(1, len(links)):
if "cart" not in links[i] or not '#' or None:
if links[i].split("/")[-1] == 'pdf':
# print "Download pdf: " + self.__url_base + links[i]
download_file(self.__session, self.__url_base + links[i], self.__config.get('path', 'path.dumps') + '/' + directory, filename + '.pdf')
elif links[i].split("/")[-1] == 'epub':
# print "Download epub: " + self.__url_base + links[i]
download_file(self.__session, self.__url_base + links[i], self.__config.get('path', 'path.dumps') + '/' + directory, filename + '.epub')
elif links[i].split("/")[-1] == 'mobi':
# print "Download mobi: " + self.__url_base + links[i]
download_file(self.__session, self.__url_base + links[i], self.__config.get('path', 'path.dumps') + '/' + directory, filename + '.mobi')
else:
# print "Download extras: " + self.__url_base + links[i]
download_file(self.__session, self.__url_base + links[i], self.__config.get('path', 'path.dumps') + '/' + directory, filename +'.zip')
wait(self.__delay)
def download_ebooks_dir(self, types,directory):
"""
"""
downloads_info = [dict(type=type,
url=self.__url_base + self.__config.get('url', 'url.download').format(self.info['book_id'], type),
filename=self.info['filename'] + '.' + type)
for type in types]
for download in downloads_info:
self.info['paths'].append(
download_file(self.__session, download['url'], directory, download['filename']))
def update_table(table, source_file, source_dir, use_cache=False):
"""Update a time series database downloading and scraping excel files.
Args:
table (dataset.Table): A table in the database used.
source_file (str): Path to an xlsx file with the list of excel files to
download and their parameters to extract data with xlseries.
source_dir (str): Path to a cache directory where excel files will be
downloaded.
use_cache (bool): When True will use the files already downloaded, when
False will re-download the files.
"""
# use pandas to read the excel where we store the metadata of the files
df_source = pd.read_excel(source_file)
# scrape every excel and add its time series to the database
for row in df_source.iterrows():
if not use_cache:
utils.download_file(row[1]["download_link"], row[1]["filename"], source_dir)
path_excel = os.path.join(source_dir, row[1]["filename"])
# here is where the magic really happens!
df_series = scrape_series(
path_excel,
row[1]["headers_coord"],
row[1]["data_starts"],
row[1]["frequency"],
row[1]["time_header_coord"],
row[1]["context"],
row[1]["ws_name"],
)
source_name = source_file.replace(".xlsx", "")
if not type(df_series) == list:
df_series = [df_series]
print(sum(map(len, df_series)), "series were scraped from", row[1]["filename"])
for df in df_series:
add_series_to_tbl(table, source_name, row[1]["categories"], row[1]["description"], df)
def _get_manager(source, venv):
"""retrieves the manager repo
This is used by default to install the installer plugins.
:param string source: url of manager tar.gz file.
:param string venv: virtualenv path.
"""
manager_tmp_dir = os.path.join(venv, 'manager')
os.makedirs(manager_tmp_dir)
tmp_tar = '{0}/tmp.tar.gz'.format(manager_tmp_dir)
lgr.debug('downloading manager code from: {0}'.format(source))
utils.download_file(source, tmp_tar)
# TODO: find a workaround for strip-components using tarfile
# TODO: check if tar before untaring
lgr.debug('extracting {0} to {1}'.format(tmp_tar, manager_tmp_dir))
utils.run('tar -xzvf {0} -C {1} --strip-components=1'.format(
tmp_tar, manager_tmp_dir))
return manager_tmp_dir
def make_collage(image_urls):
import uuid
images = []
for url in image_urls:
image_path = download_file(url)
if image_path:
images.append(image_path)
path = '/tmp/collage/{random_string}.jpeg'.format(random_string=uuid.uuid1().hex)
toCollage(images, out_file = path, collage_x = 640, collage_y = 490)
[os.remove(image_path) for image_path in images]
return path
def download_extras(self, base_path):
"""
"""
# https://github.com/niqdev/packtpub-crawler/pull/27
if self.__config.has_option('path', 'path.group'):
folder_name = self.info['title'].encode('ascii', 'ignore').replace(' ', '_') + \
self.info['author'].encode('ascii', 'ignore').replace(' ', '_')
directory = base_path + join(self.__config.get('path', 'path.ebooks'), folder_name, self.__config.get('path', 'path.extras'))
else:
directory = base_path + self.__config.get('path', 'path.extras')
url_image = self.info['url_image']
filename = self.info['filename'] + '_' + split(url_image)[1]
self.info['paths'].append(download_file(self.__session, url_image, directory, filename, self.__headers))
if 'url_source_code' in self.info:
self.info['paths'].append(download_file(self.__session, self.info['url_source_code'], directory,
self.info['filename'] + '.zip', self.__headers))
def get_chromagram(self, mode=MusicDataType.AUDIO):
"""Get chromagram for the score
Creates chromagram either from mp3,(:meth:`utils.get_chromagram_from_audio`) midi (:meth:`utils.get_chromagram_from_midi`) or echonest analysis(:meth:`apiwrappers.echonest.get_echonest_features_for_local_file`) of the score
Args:
mode: :class:`.MusicDataType`.
Returns:
Chromagram np.array
"""
directory = os.path.dirname(os.path.abspath(__file__))
base_path = directory + '/' + self.id
if mode == MusicDataType.AUDIO:
audio_file_path = base_path + '.mp3'
utils.download_file(self.mp3, audio_file_path)
chromagram = get_chromagram_from_audio(audio_file_path)
elif mode == MusicDataType.MIDI:
midi_file_path = base_path + '.mid'
utils.download_file(self.midi, midi_file_path)
chromagram = utils.get_chromagram_from_midi(midi_file_path)
elif mode == MusicDataType.ECHONEST:
audio_file_path = base_path + '.mp3'
utils.download_file(self.mp3, audio_file_path)
self.features = \
echonest.get_echonest_features_for_local_file(audio_file_path)
chromagram = self.features.chroma
np.savetxt(self.id + '.csv', chromagram)
utils.remove_file(audio_file_path)
return chromagram
def _download_files(self, url, dirs=None, files=None):
"""Download files from a github repo directory.
If no files are listed, download all the files contained in the dir"""
items = []
api_url = self._convert_url_to_api_url(url, dirs)
request = requests.get(api_url)
api_request = json.loads(request.text)
for item in api_request:
if not item['download_url']:
continue
dl_url = item['download_url']
file_path = os.path.join(self.path, item['name'])
if files:
if item['name'] in files:
items.append(file_path)
download_file(dl_url, file_path)
else:
items.append(file_path)
download_file(dl_url, file_path)
return items
def verify_tbb_tarball(tbb_ver, tarball_path, tbb_url):
tarball_filename = get_tbb_filename(tbb_ver)
tarball_sha_sum = ut.sha_256_sum_file(tarball_path).lower()
sha256sums_fname = "sha256sums"
if tbb_ver.split(".")[0] == "5":
sha256sums_fname += "-unsigned-build"
sha256sums_fname += ".txt"
sha_sum_url = "%s%s" % (get_tbb_base_url(tbb_ver), sha256sums_fname)
sha_sum_path = "%s%s" % (tarball_path, ".sha256sums.txt")
sha_sum_sig_url = "%s%s" % (sha_sum_url, ".asc")
sha_sum_sig_path = "%s%s" % (sha_sum_path, ".asc")
if not os.path.isfile(sha_sum_path):
ut.download_file(sha_sum_url, sha_sum_path)
if not os.path.isfile(sha_sum_sig_path):
ut.download_file(sha_sum_sig_url, sha_sum_sig_path)
if not verify_tbb_sig(sha_sum_sig_path):
return False
# https://github.com/micahflee/torbrowser-launcher/blob/3f1146e1a084c4e8021da968104cbc2877ae01e6/torbrowser_launcher/launcher.py#L560
for line in ut.gen_read_lines(sha_sum_path):
if tarball_sha_sum in line.lower() and tarball_filename in line:
return True
return False
def download_ebooks(self, types, base_path):
"""
"""
downloads_info = [dict(type=type,
url=self.__url_base + self.__config.get('url', 'url.download').format(self.info['book_id'], type),
filename=self.info['filename'] + '.' + type)
for type in types]
# https://github.com/niqdev/packtpub-crawler/pull/27
if self.__config.has_option('path', 'path.group'):
folder_name = self.info['title'].encode('ascii', 'ignore').replace(' ', '_') + \
self.info['author'].encode('ascii', 'ignore').replace(' ', '_')
directory = base_path + join(self.__config.get('path', 'path.ebooks'), folder_name)
else:
directory = base_path + self.__config.get('path', 'path.ebooks')
for download in downloads_info:
self.info['paths'].append(
download_file(self.__session, download['url'], directory, download['filename'], self.__headers))
def create_data_dump():
dump = {}
for sitename in SITES:
params = {"node":"{0}".format(sitename)}
fd = utils.download_file(URL, params)
stats = {}
reader = xml_reader(fd, sitename)
for block in reader:
dataset = block[1].split("#")[0]
era = get_era(dataset)
cust = 1 if block[3] == "y" else 0
ind = 0 + cust
if era not in stats:
stats[era] = {}
tier = get_tier(dataset)
if tier not in stats[era]:
stats[era][tier] = [0,0] # bytes, cust bytes
try:
stats[era][tier][ind] += block[2]
except IndexError:
print "ERROR: ", block
dump[sitename] = stats
return dump
def download_data(self, filename='', download_all=False):
if not self.isvalid:
print( 'Sorry, {}, your username or password are not match, authorization failed ...')
return
if download_all:
self.list_data()
for i in self.__all_data:
utils.download_file(self.__cookies, i)
elif type(filename) == list:
for i in filename:
utils.download_file(self.__cookies, i)
elif type(filename) == str:
utils.download_file(self.__cookies, filename)
else:
pass
def cli(verbose, debug, force, nodownload, temporal, path, name, version, level):
"""
Changesets for Feature Location
"""
logging.basicConfig(format='%(asctime)s : %(levelname)s : ' +
'%(name)s : %(funcName)s : %(message)s')
if debug:
logging.root.setLevel(level=logging.DEBUG)
elif verbose:
logging.root.setLevel(level=logging.INFO)
else:
logging.root.setLevel(level=logging.ERROR)
# load project info
projects = load_projects()
project_found = False
for project in projects:
if name == project.name:
if version and version != project.version:
continue
if level and level != project.level:
continue
project_found = True
break # got the right name/version/level
if not project_found:
error("Could not find project in projects.csv!")
logger.info("Running project on %s", str(project))
print(project)
nodownload = True # not using this just yet
if not nodownload:
logger.info("Downloading %s %s release from %s" % (project.name, project.version, project.src_url))
utils.mkdir(project.src_path)
fn = utils.download_file(project.src_url, project.src_path)
if fn.endswith('zip'):
with zipfile.ZipFile(fn, 'r') as z:
z.extractall(project.src_path)
elif fn.endswith('tar.bz2'):
with tarfile.open(fn, 'r:bz2') as z:
z.extractall(project.src_path)
elif fn.endswith('tar.gz') or fn.endswith('.tgz'):
with tarfile.open(fn, 'r:gz') as z:
z.extractall(project.src_path)
repos = load_repos(project)
# create/load document lists
queries = create_queries(project)
goldsets = load_goldsets(project)
# get corpora
changeset_corpus = create_corpus(project, repos, ChangesetCorpus, use_level=False)
release_corpus = create_release_corpus(project, repos)
# release-based evaluation is #basic 💁
release_lda, release_lsi = run_basic(project, release_corpus,
release_corpus, queries, goldsets,
'Release', use_level=True, force=force)
changeset_lda, changeset_lsi = run_basic(project, changeset_corpus,
release_corpus, queries, goldsets,
'Changeset', force=force)
if temporal:
try:
temporal_lda, temporal_lsi = run_temporal(project, repos,
changeset_corpus, queries,
goldsets, force=force)
except IOError:
logger.info("Files needed for temporal evaluation not found. Skipping.")
else:
do_science('temporal', temporal_lda, changeset_lda, ignore=True)
#do_science('temporal_lsi', temporal_lsi, changeset_lsi, ignore=True)
# do this last so that the results are printed together
do_science('basic', changeset_lda, release_lda)
import os
import zipfile
import shutil
from utils import download_file
INCHI_URL = 'http://www.inchi-trust.org/download/104/INCHI-1-API.zip'
inchi_zipfile = download_file(INCHI_URL)
zipfile.ZipFile(inchi_zipfile).extractall('.')
INCHI_SRC_DIR = os.path.join(
os.environ['SRC_DIR'], 'External', 'INCHI-API', 'src'
)
shutil.copytree('./INCHI-1-API/INCHI_API/inchi_dll', INCHI_SRC_DIR)
import os
import tarfile
import shutil
from utils import download_file
AVALON_URL = (
'http://downloads.sourceforge.net/project/avalontoolkit/'
'AvalonToolkit_1.2/AvalonToolkit_1.2.0.source.tar'
)
avalon_tarball = download_file(AVALON_URL)
AVALON_SRC_DIR = os.path.join(
os.environ['SRC_DIR'], 'External', 'AvalonTools', 'src'
)
os.makedirs(AVALON_SRC_DIR)
tarfile.open(avalon_tarball).extractall(AVALON_SRC_DIR)
# overwrite the reaccsio.c module with the patched one
RECIPE_DIR = os.path.dirname(__file__)
REACCSIO_SRC = os.path.join(RECIPE_DIR, 'avalon_reaccsio.c')
REACCSIO_DST = os.path.join(
AVALON_SRC_DIR, 'SourceDistribution', 'common', 'reaccsio.c'
)
shutil.copyfile(REACCSIO_SRC, REACCSIO_DST)
def action_deploy():
""" Update user file in the deployment environment.
"""
app = make_app(config=DEPLOY_CFG)
download_file(app.config['USERS_URL'], app.config['USERS_XML'])
def action_debug():
""" Update user file on development machine.
"""
app = make_app(config=DEBUG_CFG)
download_file(app.config['USERS_URL'], app.config['USERS_XML'])
