def _create_test_rings(path):
testgz = os.path.join(path, 'object.ring.gz')
intended_replica2part2dev_id = [
[0, 1, 2, 3, 4, 5, 6],
[1, 2, 3, 0, 5, 6, 4],
[2, 3, 0, 1, 6, 4, 5],
]
intended_devs = [
{'id': 0, 'device': 'sda', 'zone': 0, 'ip': '127.0.0.0', 'port': 6000},
{'id': 1, 'device': 'sda', 'zone': 1, 'ip': '127.0.0.1', 'port': 6000},
{'id': 2, 'device': 'sda', 'zone': 2, 'ip': '127.0.0.2', 'port': 6000},
{'id': 3, 'device': 'sda', 'zone': 4, 'ip': '127.0.0.3', 'port': 6000},
{'id': 4, 'device': 'sda', 'zone': 5, 'ip': '127.0.0.4', 'port': 6000},
{'id': 5, 'device': 'sda', 'zone': 6,
'ip': 'fe80::202:b3ff:fe1e:8329', 'port': 6000},
{'id': 6, 'device': 'sda', 'zone': 7,
'ip': '2001:0db8:85a3:0000:0000:8a2e:0370:7334', 'port': 6000},
]
intended_part_shift = 30
with closing(GzipFile(testgz, 'wb')) as f:
pickle.dump(
ring.RingData(intended_replica2part2dev_id,
intended_devs, intended_part_shift),
f)
testgz = os.path.join(path, 'object-1.ring.gz')
with closing(GzipFile(testgz, 'wb')) as f:
pickle.dump(
ring.RingData(intended_replica2part2dev_id,
intended_devs, intended_part_shift),
f)
return
def iter_hashes(self, name='RECORD'):
'''Iterate over the files and hashes of a RECORD file.
The RECORD file with the given name will be iterated over
yielding a three tuple with each iteration: filename (relative
to the package), computed hash (just calculated), and expected
hash (from RECORD file).
'''
hashless = [posixpath.join(self.dist_info, name + ext)
for ext in ['', '.jws', '.p7s']]
path = posixpath.join(self.dist_info, name)
with closing(self.open(path)) as record_file:
for row in csv.reader(record_file):
filename, hashspec = row[:2]
if not hashspec:
if filename not in hashless:
yield filename, None, None
continue
algo, expected_hash = hashspec.split('=', 1)
hash = hashlib.new(algo)
with closing(self.open(filename, 'rb')) as file:
while True:
data = file.read(4096)
if not data:
break
hash.update(data)
hash = base64.urlsafe_b64encode(hash.digest()).rstrip('=')
yield filename, hash, expected_hash
def _fake_rings(tmpdir):
account_ring_path = os.path.join(tmpdir, 'account.ring.gz')
with closing(GzipFile(account_ring_path, 'wb')) as f:
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6012},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6022}], 30),
f)
container_ring_path = os.path.join(tmpdir, 'container.ring.gz')
with closing(GzipFile(container_ring_path, 'wb')) as f:
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6011},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6021}], 30),
f)
object_ring_path = os.path.join(tmpdir, 'object.ring.gz')
with closing(GzipFile(object_ring_path, 'wb')) as f:
pickle.dump(ring.RingData([[0, 1, 0, 1], [1, 0, 1, 0]],
[{'id': 0, 'zone': 0, 'device': 'sda1', 'ip': '127.0.0.1',
'port': 6010},
{'id': 1, 'zone': 1, 'device': 'sdb1', 'ip': '127.0.0.1',
'port': 6020}], 30),
f)
def test_ascii():
""" Test ascii/unicode bar """
# Test ascii autodetection
with closing(StringIO()) as our_file:
with tqdm(total=10, file=our_file, ascii=None) as t:
assert t.ascii # TODO: this may fail in the future
# Test ascii bar
with closing(StringIO()) as our_file:
for _ in tqdm(_range(3), total=15, file=our_file, miniters=1,
mininterval=0, ascii=True):
pass
our_file.seek(0)
res = our_file.read().strip("\r").split("\r")
assert '7%|6' in res[1]
assert '13%|#3' in res[2]
assert '20%|##' in res[3]
# Test unicode bar
with closing(UnicodeIO()) as our_file:
with tqdm(total=15, file=our_file, ascii=False, mininterval=0) as t:
for _ in _range(3):
t.update()
our_file.seek(0)
res = our_file.read().strip("\r").split("\r")
assert "7%|\u258b" in res[1]
assert "13%|\u2588\u258e" in res[2]
assert "20%|\u2588\u2588" in res[3]
def test_list_inconsistent_obj(ctx, manager, osd_remote, pg, acting, osd_id,
obj_name, obj_path):
mon = manager.controller
pool = 'rbd'
omap_key = 'key'
omap_val = 'val'
manager.do_rados(mon, ['-p', pool, 'setomapval', obj_name,
omap_key, omap_val])
messup = MessUp(manager, osd_remote, pool, osd_id, obj_name, obj_path,
omap_key, omap_val)
for test in [messup.rm_omap, messup.add_omap, messup.change_omap,
messup.append, messup.truncate, messup.change_obj,
messup.remove]:
with test() as checks:
deep_scrub(manager, pg)
cmd = 'rados list-inconsistent-pg {pool} ' \
'--format=json'.format(pool=pool)
with contextlib.closing(StringIO()) as out:
mon.run(args=cmd.split(), stdout=out)
pgs = json.loads(out.getvalue())
assert pgs == [pg]
cmd = 'rados list-inconsistent-obj {pg} ' \
'--format=json'.format(pg=pg)
with contextlib.closing(StringIO()) as out:
mon.run(args=cmd.split(), stdout=out)
objs = json.loads(out.getvalue())
assert len(objs['inconsistents']) == 1
checker = InconsistentObjChecker(osd_id, acting, obj_name)
inc_obj = objs['inconsistents'][0]
log.info('inc = %r', inc_obj)
checker.basic_checks(inc_obj)
for check in checks:
checker.run(check, inc_obj)
def unarchive(archive_path, dest):
"""Extract the contents of a tar or zip file at *archive_path* into the
directory *dest*.
:type archive_path: str
:param archive_path: path to archive file
:type dest: str
:param dest: path to directory where archive will be extracted
*dest* will be created if it doesn't already exist.
tar files can be gzip compressed, bzip2 compressed, or uncompressed. Files
within zip files can be deflated or stored.
"""
if tarfile.is_tarfile(archive_path):
with contextlib.closing(tarfile.open(archive_path, 'r')) as archive:
archive.extractall(dest)
elif zipfile.is_zipfile(archive_path):
with contextlib.closing(zipfile.ZipFile(archive_path, 'r')) as archive:
for name in archive.namelist():
# the zip spec specifies that front slashes are always
# used as directory separators
dest_path = os.path.join(dest, *name.split('/'))
# now, split out any dirname and filename and create
# one and/or the other
dirname, filename = os.path.split(dest_path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
if filename:
with open(dest_path, 'wb') as dest_file:
dest_file.write(archive.read(name))
else:
raise IOError('Unknown archive type: %s' % (archive_path,))
def database(timestamp, channel, epg, out_file):
rec_date = datetime.datetime.fromtimestamp(
int(timestamp)).strftime('%Y.%m.%d')
rec_time = datetime.datetime.fromtimestamp(
int(timestamp)).strftime('%H:%M')
pubdate = formatdate(time.time(), True)
title = epg[1]
if epg[2] == '':
description = epg[3]
else:
description = epg[2] + '\n\n' + epg[3]
audiofile = os.path.basename(out_file)
length, length_bytes = audiolength(out_file)
with closing(MySQLdb.connect(
login.DB_HOST, login.DB_USER,
login.DB_PASSWORD, login.DB_DATABASE)) as connection:
with closing(connection.cursor()) as cursor:
cursor.execute('INSERT INTO recordings \
(date, time, channel, title, description, audiofile, timestamp, \
length, bytes, pubdate) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)',
(rec_date, rec_time, channel, title, description,
audiofile, timestamp, length, length_bytes, pubdate))
connection.commit()
def name_to_config(template):
"""Read template file into a dictionary to use as base for all samples.
Handles well-known template names, pulled from GitHub repository and local
files.
"""
if objectstore.is_remote(template):
with objectstore.open(template) as in_handle:
config = yaml.load(in_handle)
with objectstore.open(template) as in_handle:
txt_config = in_handle.read()
elif os.path.isfile(template):
if template.endswith(".csv"):
raise ValueError("Expected YAML file for template and found CSV, are arguments switched? %s" % template)
with open(template) as in_handle:
txt_config = in_handle.read()
with open(template) as in_handle:
config = yaml.load(in_handle)
else:
base_url = "https://raw.github.com/chapmanb/bcbio-nextgen/master/config/templates/%s.yaml"
try:
with contextlib.closing(urllib2.urlopen(base_url % template)) as in_handle:
txt_config = in_handle.read()
with contextlib.closing(urllib2.urlopen(base_url % template)) as in_handle:
config = yaml.load(in_handle)
except (urllib2.HTTPError, urllib2.URLError):
raise ValueError("Could not find template '%s' locally or in standard templates on GitHub"
% template)
return config, txt_config
def api_request_native(url, data=None, method=None):
request = urllib2.Request(url)
if method:
request.get_method = lambda: method
try:
request.add_header('Authorization', 'token ' + token_auth_string())
except MissingTokenException:
request.add_header('Authorization', 'Basic ' + base64.urlsafe_b64encode(basic_auth_string()))
request.add_header('Accept', 'application/json')
request.add_header('Content-Type', 'application/json')
if data is not None:
request.add_data(data)
if settings.get('https_proxy'):
opener = urllib2.build_opener(urllib2.HTTPHandler(), urllib2.HTTPSHandler(),
urllib2.ProxyHandler({'https': settings.get('https_proxy')}))
urllib2.install_opener(opener)
try:
with contextlib.closing(urllib2.urlopen(request)) as response:
if response.code == 204: # No Content
return None
else:
return json.loads(response.read())
except urllib2.HTTPError as err:
with contextlib.closing(err):
raise SimpleHTTPError(err.code, err.read())
def handle_ds_client(reader, writer, key):
with closing(writer):
loop = reader._loop
bind_address = yield from get_host_name(writer)
# Connect to client
connection = yield from get_connection(key, loop)
if not connection:
return
ds_reader, ds_writer = connection
# Loop over reply/requests
with closing(ds_writer):
while not reader.at_eof() and not ds_reader.at_eof():
# Read request
request = yield from forward_giop_frame(
reader, ds_writer, bind_address)
if not request:
break
# Choose patch
if ZMQ_SUBSCRIPTION_CHANGE in request:
patch = Patch.ZMQ
else:
patch = Patch.NONE
# Read reply_header
reply = yield from forward_giop_frame(
ds_reader, writer, bind_address, patch=patch)
def setUp(self):
self.TOO_LONG_RAW = ''.join(random.choice(string.ascii_uppercase)
for _ in range(2*Util.MAX_BYTES))
with closing(StringIO()) as buf:
with closing(GzipFile('', 'wb', 9, buf)) as f:
f.write(self.TOO_LONG_RAW)
self.TOO_LONG_GZIP = base64.b64encode(buf.getvalue())
def get_quote_info(ctx, pline, userdata):
args = pline.trailing.split()
args.pop(0)
if args:
try:
quote_id = int(args.pop(0))
except ValueError:
ctx.command('/say Invalid argument.')
return
else:
ctx.command('/say You must provide a quote id.')
return
init_quotes_db()
with closing(sqlite3.connect(quote_db_filename)) as quote_db:
with closing(quote_db.cursor()) as cursor:
cursor.execute('''SELECT "id", "server", "channel", "date" FROM "quotes" WHERE "id"=?''', (quote_id,))
row = cursor.fetchone()
if row is None:
ctx.command('/say That quote does not exist.')
return
quote_id, quote_server, quote_channel, quote_date = row
date, time = quote_date.split()
year, month, day = map(int, date.split('-'))
hour, minute, second = map(int, time.split(':'))
dt_utc_naive = datetime(year, month, day, hour, minute, second)
dt = pytz.timezone('Europe/Berlin').fromutc(dt_utc_naive)
ctx.command(u'/say \x02#{} infos:\x02 {} \x02@\x02 {} \x02at\x02 {}'.format(
quote_id, quote_channel, quote_server, dt.strftime(u'%Y-%m-%d %H:%M')
).encode('utf-8'))
def handle_db_client(reader, writer, key):
with closing(writer):
loop = reader._loop
bind_address = yield from get_host_name(writer)
# Connect to client
connection = yield from get_connection(key, loop)
if not connection:
return
db_reader, db_writer = connection
# Loop over reply/requests
with closing(db_writer):
while not reader.at_eof() and not db_reader.at_eof():
# Read request
request = yield from forward_giop_frame(
reader, db_writer, bind_address)
if not request:
break
# Choose patch
if IMPORT_DEVICE in request:
patch = Patch.IOR
elif GET_CSDB_SERVER in request:
patch = Patch.CSD
else:
patch = Patch.NONE
# Read reply_header
reply = yield from forward_giop_frame(
db_reader, writer, bind_address, patch=patch)
def test_get_job(conn):
job_source_name = "dummy-job-src"
path = "/data/kpi_2.csv"
job_type = 'dummy'
filesize = 1060
description = {"uri": path}
description_json = json.dumps(description)
with closing(conn.cursor()) as cursor:
job_source_id = add_job_source(cursor, job_source_name, "dummy", '{}')
enqueue_job(conn, job_type, description_json, filesize, job_source_id)
conn.commit()
with closing(conn.cursor()) as cursor:
job = get_job(cursor)
conn.commit()
_, job_type, description, _, _ = job
eq_(path, json.loads(description)["uri"])
with closing(conn.cursor()) as cursor:
job = get_job(cursor)
eq_(job, None)
def add_quote(ctx, pline, userdata):
trailing = pline.trailing.split(None, 1)
if len(trailing) <= 1:
ctx.command('/say No quote given.')
return
_, quote = trailing
quote = try_decode(quote)
server = try_decode(ctx.server)
channel = try_decode(ctx.channel)
init_quotes_db()
with closing(sqlite3.connect(quote_db_filename)) as quote_db:
with closing(quote_db.cursor()) as cursor:
cursor.execute(u'''
INSERT INTO quotes
("line", "server", "channel", "date")
VALUES
(?, ?, ?, datetime('now', 'utc'))
''', (quote, server, channel)
)
insert_id = cursor.lastrowid
quote_db.commit()
ctx.command('/say Quote inserted (#{})'.format(insert_id))
def add_packages(self, connection, repository, packages):
basedir = utils.get_path_from_url(repository.url)
index_file = utils.get_path_from_url(
self._get_url_of_metafile(repository, "Packages")
)
utils.ensure_dir_exist(os.path.dirname(index_file))
index_gz = index_file + ".gz"
count = 0
# load existing packages
self.get_packages(connection, repository, packages.add)
with open(index_file, "wb") as fd1:
with closing(gzip.open(index_gz, "wb")) as fd2:
writer = utils.composite_writer(fd1, fd2)
for pkg in packages:
filename = os.path.join(basedir, pkg.filename)
with closing(debfile.DebFile(filename)) as deb:
debcontrol = deb.debcontrol()
debcontrol.setdefault("Origin", repository.origin)
debcontrol["Size"] = str(pkg.filesize)
debcontrol["Filename"] = pkg.filename
for k, v in six.moves.zip(_CHECKSUM_METHODS, pkg.checksum):
debcontrol[k] = v
writer(debcontrol.dump())
writer("\n")
count += 1
self.logger.info("saved %d packages in %s", count, repository)
self._update_suite_index(repository)
def test_waiting_locks(conn):
job_source_name = "dummy-job-src"
path = "/data/kpi_2.csv"
job_type = 'dummy'
filesize = 1060
description_json = '{{"uri": "{}"}}'.format(path)
with closing(conn.cursor()) as cursor:
datasource = name_to_datasource(cursor, "dummy-src")
job_source_id = get_job_source(cursor, job_source_name)
if not job_source_id:
job_source_id = add_job_source(cursor, job_source_name, "dummy", '{}')
enqueue_job(conn, job_type, description_json, filesize, job_source_id)
conn.commit()
with closing(conn.cursor()) as cursor:
job = get_job(cursor)
conn.commit()
with closing(conn.cursor()) as cursor:
cursor.execute("select relation::regclass, * FROM pg_locks WHERE NOT GRANTED")
rows = cursor.fetchall()
job_id, job_type, description, size, parser_config = job
eq_(path, json.loads(description)["uri"])
def recommend(self, uid, comment, title=None, t=None, ck=None):
"""recommend a uid with some comment. ck is optional, if
not provided, we will try to fetch a ck."""
t = t or 'W'
if ck is None:
## get recommend ck
url = "http://www.douban.com/j/recommend?type=%s&uid=%s&rec=" % (t,uid)
with contextlib.closing(httplib.HTTPConnection("music.douban.com")) as conn:
cookie = 'dbcl2="%s"; bid="%s"; ' % (self.dbcl2, self.bid)
conn.request('GET', url, None, {'Cookie': cookie})
result = conn.getresponse().read()
ck = self.__parse_ck(result)
if ck:
post = {'ck':ck, 'comment':comment, 'novote':1, 'type':t, 'uid':uid}
if title:
post['title'] = title
## convert unicode chars to bytes
data = urllib.urlencode(post)
## ck ?
cookie = 'dbcl2="%s"; bid="%s"; ck=%s' % (self.dbcl2, self.bid, ck)
accept = 'application/json'
content_type= 'application/x-www-form-urlencoded; charset=UTF-8'
header = {"Cookie": cookie, "Accept": accept,
"Content-Type":content_type, }
with contextlib.closing(httplib.HTTPConnection("www.douban.com")) as conn:
conn.request('POST', "/j/recommend", data, header)
conn.getresponse().read()
def start(self, *args, **kws):
with\
closing(socket.socket( socket.AF_INET,
socket.SOCK_RAW, socket.getprotobyname('icmp') )) as self.ipv4,\
closing(socket.socket( socket.AF_INET6,
socket.SOCK_RAW, socket.getprotobyname('ipv6-icmp') )) as self.ipv6:
return self._start(*args, **kws)
def test_uncompress():
# Create dummy file
fd, temp = mkstemp()
os.close(fd)
# Create a zipfile
dtemp = mkdtemp()
ztemp = os.path.join(dtemp, 'test.zip')
with contextlib.closing(zipfile.ZipFile(ztemp, 'w')) as testzip:
testzip.write(temp)
datasets.utils._uncompress_file(ztemp, verbose=0)
assert(os.path.exists(os.path.join(dtemp, temp)))
shutil.rmtree(dtemp)
dtemp = mkdtemp()
ztemp = os.path.join(dtemp, 'test.tar')
with contextlib.closing(tarfile.open(ztemp, 'w')) as tar:
tar.add(temp)
datasets.utils._uncompress_file(ztemp, verbose=0)
assert(os.path.exists(os.path.join(dtemp, temp)))
shutil.rmtree(dtemp)
dtemp = mkdtemp()
ztemp = os.path.join(dtemp, 'test.gz')
f = gzip.open(ztemp, 'wb')
f.close()
datasets.utils._uncompress_file(ztemp, verbose=0)
assert(os.path.exists(os.path.join(dtemp, temp)))
shutil.rmtree(dtemp)
os.remove(temp)
def getHist(self):
with closing (ocpcaproj.OCPCAProjectsDB()) as projdb:
proj = projdb.loadToken(self.token)
with closing (ocpcadb.OCPCADB(proj)) as db:
ch = proj.getChannelObj(self.channel)
# Get the source database sizes
[[ximagesz, yimagesz, zimagesz], timerange] = proj.datasetcfg.imageSize(self.res)
[xcubedim, ycubedim, zcubedim] = cubedim = proj.datasetcfg.getCubeDims()[self.res]
[xoffset, yoffset, zoffset] = proj.datasetcfg.getOffset()[self.res]
# Set the limits for iteration on the number of cubes in each dimension
xlimit = (ximagesz-1) / xcubedim + 1
ylimit = (yimagesz-1) / ycubedim + 1
zlimit = (zimagesz-1) / zcubedim + 1
hist_sum = np.zeros(self.numbins, dtype=np.uint32)
# sum the histograms
for z in range(zlimit):
for y in range(ylimit):
for x in range(xlimit):
# cutout the data for the cube
data = db.cutout(ch, [ x*xcubedim, y*ycubedim, z*zcubedim], cubedim, self.res ).data
# compute the histogram and store it
(hist, bins) = np.histogram(data[data > 0], bins=self.numbins, range=(0,self.numbins))
hist_sum = np.add( hist_sum, hist )
print "Processed cube {} {} {}".format(x,y,z)
return (hist_sum, bins)
def _cache(fn, *args, **kwargs):
if cache.disabled:
return fn(*args, **kwargs)
# A bit obscure, but simplest way to generate unique key for
# functions and methods in python 2 and 3:
key = "{}.{}".format(fn.__module__, repr(fn).split("at")[0])
etag = ".".join(_get_mtime(name) for name in depends_on)
cache_path = _get_cache_path()
try:
with closing(shelve.open(cache_path)) as db:
if db.get(key, {}).get("etag") == etag:
return db[key]["value"]
else:
value = fn(*args, **kwargs)
db[key] = {"etag": etag, "value": value}
return value
except shelve_open_errors:
# Caused when going from Python 2 to Python 3 and vice-versa
warn("Removing possibly out-dated cache")
os.remove(cache_path)
with closing(shelve.open(cache_path)) as db:
value = fn(*args, **kwargs)
db[key] = {"etag": etag, "value": value}
return value
def write_easy_install_pth(self, exts):
paths = []
for ext in sorted(exts.values()):
ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir)
easy_pth = "%s/easy-install.pth" % ext_site_packages
if not os.path.isfile(easy_pth):
continue
with closing(open(easy_pth)) as f:
for line in f:
line = line.rstrip()
# Skip lines matching these criteria
if not line: continue
if re.search(r'^(import|#)', line): continue
if (ext.name != 'py-setuptools' and
re.search(r'setuptools.*egg$', line)): continue
paths.append(line)
site_packages = os.path.join(self.prefix, self.site_packages_dir)
main_pth = "%s/easy-install.pth" % site_packages
if not paths:
if os.path.isfile(main_pth):
os.remove(main_pth)
else:
with closing(open(main_pth, 'w')) as f:
f.write("import sys; sys.__plen = len(sys.path)\n")
for path in paths:
f.write("%s\n" % path)
f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; "
"p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n")
def download_from_url(url, destination_path, force=False, aspera=False):
"""Download file from remote server
:param url: path to the file on remote server (including file name)
:param destination_path: path to the file on local machine (including file name)
:param force: bool - if file exist force to overwrite it , defaults to False
"""
try:
is_already_downloaded = os.path.isfile(destination_path)
if is_already_downloaded:
if force:
with closing(urlopen(url)) as r:
with open(destination_path, mode='wb') as f:
stderr.write("Downloading %s to %s\n" % (url, destination_path))
copyfileobj(r, f)
else:
stderr.write("File already exist. Use force=True if you would like to overwrite it.\n")
else:
if aspera:
download_aspera(url, destination_path)
else:
with closing(urlopen(url)) as r:
with open(destination_path, mode='wb') as f:
stderr.write("Downloading %s to %s\n" % (url, destination_path))
copyfileobj(r, f)
except URLError:
stderr.write("Cannot find file %s" % url)
def _play_proxy(self, media, player_name, args):
"""
Load data with python requests and pipe data to a media player.
We need this function for url that use redirection and cookies.
This function is used if the non-standard,
non-API compliant '_play_proxy' attribute of the 'media' object is defined and is True.
"""
if args is None:
for (binary, stdin_args) in PLAYERS:
if binary == player_name:
args = stdin_args
assert args is not None
print(':: Play_proxy streaming from %s' % media.url)
print(':: to %s %s' % (player_name, args))
print(player_name + ' ' + args)
proc = Popen(player_name + ' ' + args, stdin=PIPE, shell=True)
# Handle cookies (and redirection 302...)
session = requests.sessions.Session()
with closing(proc.stdin):
with closing(session.get(media.url, stream=True)) as response:
for buffer in response.iter_content(8192):
try:
proc.stdin.write(buffer)
except:
print("play_proxy broken pipe. Can't write anymore.")
break
def test_entrypoint(self):
# Test that the entrypoint fetching also works (even with dialects)
# using the same configuration we used in setUp() but not using
# the impl_sqlalchemy SQLAlchemyBackend class directly...
with contextlib.closing(backends.fetch(self.db_conf)) as backend:
with contextlib.closing(backend.get_connection()):
pass
def _cache(fn, *args, **kwargs):
if cache.disabled:
return fn(*args, **kwargs)
# A bit obscure, but simplest way to generate unique key for
# functions and methods in python 2 and 3:
key = '{}.{}'.format(fn.__module__, repr(fn).split('at')[0])
etag = '.'.join(_get_mtime(name) for name in depends_on)
cache_path = _get_cache_path()
try:
with closing(shelve.open(cache_path)) as db:
if db.get(key, {}).get('etag') == etag:
return db[key]['value']
else:
value = fn(*args, **kwargs)
db[key] = {'etag': etag, 'value': value}
return value
except dbm.error:
# Caused when going from Python 2 to Python 3
warn("Removing possibly out-dated cache")
os.remove(cache_path)
with closing(shelve.open(cache_path)) as db:
value = fn(*args, **kwargs)
db[key] = {'etag': etag, 'value': value}
return value
def generate_index(self):
"""Generated the repository index file (`index.gz`)
index.gz is the compressed sqlite index containing all of the succeeded
packages in the repository pool.
Return the number of packages added to the repository index.
"""
assert exists(self.repository.path)
idx_path = xjoin(self.repository.path, 'index')
idx_gz_path = idx_path + '.gz'
sh.rm(idx_path)
db = RepoPackageDatabase(idx_path, touch=True)
# Tag BE packages; so client may use it to determine if a package is
# available only to BE customers or not.
# See also: RepoPackage.requires_be_license property
pkgtags = 'be' if self.repository.name == 'be' else ''
with closing(db):
LOG.debug('finding packages in %s', self.repository.path)
packages = self.repository.find_packages()
LOG.debug('processing %d packages', len(packages))
rpkg_list = [
RepoPackage.create_from(
BinaryPackage(**self._read_info_json(pkgfile)),
relpath=relpath(pkgfile, self.repository.path),
tags=pkgtags)
for pkgfile in textui.ProgressBar.iterate(packages, note="Package")
]
# Optimize index size by removing the "description" field.
# PyPI's descriptions are typically very long - see
# http://pypi.python.org/pypi/zc.buildout for example - hence we
# must remove them from the index.
for rpkg in rpkg_list:
rpkg.description = ''
# keep only the latest pkg_version in index
LOG.debug("pruning older pkg_version's")
rpkg_list = _prune_older_binary_releases(rpkg_list)
LOG.debug('.. resulting in %d packages', len(rpkg_list))
LOG.info(' writing index (please wait) ...')
with db.transaction() as session:
session.add_all(rpkg_list)
session.commit()
session.close()
LOG.info(' compressing index: ...%s%s',
os.path.basename(idx_gz_path),
(' (%d)' % len(rpkg_list)) if rpkg_list else '')
sh.rm(idx_gz_path)
with closing(gzip.open(idx_gz_path, 'wb')) as f:
f.write(open(idx_path, 'rb').read())
sh.rm(idx_path)
return len(rpkg_list)
def post(self):
userid = SessionUtil.get_user_id(request.form['session'])
if not userid:
return {'status': ServerConstants.STATUS_INVALID_SESSION}
eventid = request.form['eventid']
with closing(ServerConstants.mysql_pool.get_connection()) as db:
with closing(db.cursor()) as cursor:
sql = "SELECT filename, type FROM `events` WHERE id='{}'".format(eventid)
cursor.execute(sql)
rows = cursor.fetchall()
if rows:
filename = ServerConstants.STORAGE_DIR+rows[0][0]
event_type = rows[0][1]
if event_type == 'jpeg':
#output = StringIO()
#img = Image.open(filename)
#img.save(output, 'JPEG')
#output.seek(0)
#return send_file(output, mimetype='image/jpeg')
return send_file(filename, mimetype='image/jpeg')
elif event_type == 'ogg':
return send_file(filename, mimetype='audio/ogg')
elif event_type == 'h264':
return send_file(filename, mimetype='video/h264')
def _cache(fn, *args, **kwargs):
if cache.disabled:
return fn(*args, **kwargs)
# A bit obscure, but simplest way to generate unique key for
# functions and methods in python 2 and 3:
key = '{}.{}'.format(fn.__module__, repr(fn).split('at')[0])
etag = '.'.join(_get_mtime(name) for name in depends_on)
cache_dir = get_cache_dir()
cache_path = Path(cache_dir).joinpath('thefuck').as_posix()
try:
with closing(shelve.open(cache_path)) as db:
if db.get(key, {}).get('etag') == etag:
return db[key]['value']
else:
value = fn(*args, **kwargs)
db[key] = {'etag': etag, 'value': value}
return value
except (shelve_open_error, ImportError):
# Caused when switching between Python versions
warn("Removing possibly out-dated cache")
os.remove(cache_path)
with closing(shelve.open(cache_path)) as db:
value = fn(*args, **kwargs)
db[key] = {'etag': etag, 'value': value}
return value
def establish_host_connectivity():
"""Continually try to recreate the SSH Tunnels to all hosts for 2 minutes
"""
return closing(
TunnelCollection(ssh_user, ssh_key_path, host_list_w_port))
def initialize(self, put_timestamp=None, storage_policy_index=None):
"""
Create the DB
The storage_policy_index is passed through to the subclass's
``_initialize`` method. It is ignored by ``AccountBroker``.
:param put_timestamp: internalized timestamp of initial PUT request
:param storage_policy_index: only required for containers
"""
if self.db_file == ':memory:':
tmp_db_file = None
conn = get_db_connection(self.db_file, self.timeout)
else:
mkdirs(self.db_dir)
fd, tmp_db_file = mkstemp(suffix='.tmp', dir=self.db_dir)
os.close(fd)
conn = sqlite3.connect(tmp_db_file, check_same_thread=False,
factory=GreenDBConnection, timeout=0)
# creating dbs implicitly does a lot of transactions, so we
# pick fast, unsafe options here and do a big fsync at the end.
with closing(conn.cursor()) as cur:
cur.execute('PRAGMA synchronous = OFF')
cur.execute('PRAGMA temp_store = MEMORY')
cur.execute('PRAGMA journal_mode = MEMORY')
conn.create_function('chexor', 3, chexor)
conn.row_factory = sqlite3.Row
conn.text_factory = str
conn.executescript("""
CREATE TABLE outgoing_sync (
remote_id TEXT UNIQUE,
sync_point INTEGER,
updated_at TEXT DEFAULT 0
);
CREATE TABLE incoming_sync (
remote_id TEXT UNIQUE,
sync_point INTEGER,
updated_at TEXT DEFAULT 0
);
CREATE TRIGGER outgoing_sync_insert AFTER INSERT ON outgoing_sync
BEGIN
UPDATE outgoing_sync
SET updated_at = STRFTIME('%s', 'NOW')
WHERE ROWID = new.ROWID;
END;
CREATE TRIGGER outgoing_sync_update AFTER UPDATE ON outgoing_sync
BEGIN
UPDATE outgoing_sync
SET updated_at = STRFTIME('%s', 'NOW')
WHERE ROWID = new.ROWID;
END;
CREATE TRIGGER incoming_sync_insert AFTER INSERT ON incoming_sync
BEGIN
UPDATE incoming_sync
SET updated_at = STRFTIME('%s', 'NOW')
WHERE ROWID = new.ROWID;
END;
CREATE TRIGGER incoming_sync_update AFTER UPDATE ON incoming_sync
BEGIN
UPDATE incoming_sync
SET updated_at = STRFTIME('%s', 'NOW')
WHERE ROWID = new.ROWID;
END;
""")
if not put_timestamp:
put_timestamp = Timestamp(0).internal
self._initialize(conn, put_timestamp,
storage_policy_index=storage_policy_index)
conn.commit()
if tmp_db_file:
conn.close()
with open(tmp_db_file, 'r+b') as fp:
os.fsync(fp.fileno())
with lock_parent_directory(self.db_file, self.pending_timeout):
if os.path.exists(self.db_file):
# It's as if there was a "condition" where different parts
# of the system were "racing" each other.
raise DatabaseAlreadyExists(self.db_file)
renamer(tmp_db_file, self.db_file)
self.conn = get_db_connection(self.db_file, self.timeout)
else:
self.conn = conn
def download_segment(segment_url, dash_folder):
""" Module to download the segment """
#parse_url = urlparse.urlparse(segment_url)
#connection = HTTPConnectionPool(parse_url.netloc)
#chunk_dl_rates = []
parsed_uri = urlparse.urlparse(segment_url)
segment_path = '{uri.path}'.format(uri=parsed_uri)
while segment_path.startswith('/'):
segment_path = segment_path[1:]
segment_filename = os.path.join(dash_folder,
os.path.basename(segment_path))
make_sure_path_exists(os.path.dirname(segment_filename))
#segment_file_handle = open(segment_filename, 'wb')
chunk_dl_rates = []
segment_size = 0
try:
#print segment_url
total_data_dl_time = 0
chunk_number = 0
chunk_start_time = timeit.default_timer()
with closing(connection.get(segment_url, stream=True)) as seg_conn:
with open(segment_filename, 'wb') as segment_file_handle:
for segment_data in seg_conn.iter_content(DOWNLOAD_CHUNK):
if segment_data is None:
break
segment_file_handle.write(segment_data)
segment_size += len(segment_data)
if len(segment_data) < DOWNLOAD_CHUNK:
timenow = timeit.default_timer()
chunk_dl_time = timenow - chunk_start_time
chunk_number += 1
total_data_dl_time += chunk_dl_time
current_chunk_dl_rate = segment_size * 8 / total_data_dl_time
chunk_dl_rates.append(current_chunk_dl_rate)
with open(
'/mnt/QUIClientServer0/chunk_rate_iter_squad_requests_HTTP1.txt',
'a') as chk:
chk.write("%s" % segment_url)
for item in chunk_dl_rates:
chk.write(",%s" % item)
chk.write("\n")
#print chunk_dl_rates
#print "-----------------"
#print segment_w_chunks
#print "!!!!!!!!!!!!!!!!!"
segment_w_chunks.append(chunk_dl_rates)
#print segment_w_chunks
#print "##################"
break
timenow = timeit.default_timer()
chunk_dl_time = timenow - chunk_start_time
total_data_dl_time += chunk_dl_time
current_chunk_dl_rate = segment_size * 8 / total_data_dl_time
chunk_start_time = timenow
chunk_number += 1
chunk_dl_rates.append(current_chunk_dl_rate)
except urllib2.HTTPError, error:
config_dash.LOG.error(
"Unable to download DASH Segment {} HTTP Error:{} ".format(
segment_url, str(error.code)))
return None
#importing librarries
import wave, math, contextlib
import speech_recognition as sr
from moviepy.editor import AudioFileClip
#extracting the audio from the video file. hard coded the file names for now
transcribed_audio_file_name = "transcribed_speech.wav"
zoom_video_file_name = "Fork_tutorial.mp4"
audioclip = AudioFileClip(zoom_video_file_name)
audioclip.write_audiofile(transcribed_audio_file_name)
#finding out the total duration of the audio clip
with contextlib.closing(wave.open(transcribed_audio_file_name,'r')) as f:
frames = f.getnframes()
rate = f.getframerate()
duration = frames / float(rate)
total_duration = math.ceil(duration / 60)
#transcribing the audio in to text in chunks of 10 MB (limitation of the api call)
r = sr.Recognizer()
for i in range(0, total_duration):
with sr.AudioFile(transcribed_audio_file_name) as source:
audio = r.record(source, offset=i*60, duration=60)
f = open("transcription.txt", "a")
f.write(r.recognize_google(audio))
f.write(" ")
f.close()
link = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
default_video = True
if not default_video:
chrome_options.add_argument("--mute-audio")
amount = 3
try:
amount = int(input("Amount of reloads to search: "))
except ValueError:
amount = 3
errors = 1
try:
errors = int(input("Amount of errors to be displayed: "))
except ValueError:
errors = 1
with closing(Chrome(chrome_options=chrome_options)) as driver:
wait = WebDriverWait(driver, 50)
if sys.platform == "win32":
os.system("cls")
else:
os.system("clear")
req = Request(link, headers={'User-Agent': 'Mozilla/5.0'})
webpage = urlopen(req).read()
# Creating a BeautifulSoup object of the html page for easy extraction of data.
soup = BeautifulSoup(webpage, 'html.parser')
html = soup.prettify('utf-8')
if not default_video:
def init_db():
with closing(connect_db()) as db:
with app.open_resource('schema.sql', mode='r') as f:
db.cursor().executescript(f.read())
db.commit()
def main():
"""
Install system packages:
postgresql postgis osm2pgsql
"""
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
assert shutil.which('osm2pgsql'), 'install osm2pgsql'
dbname = 'lietuva'
database_created = False
with contextlib.closing(psycopg2.connect('postgresql:///postgres')) as conn:
conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
with contextlib.closing(conn.cursor()) as cur:
cur.execute("SELECT EXISTS(SELECT * FROM pg_database WHERE datname=%s)", (dbname,))
exists, = cur.fetchone()
if not exists:
logger.info("create database %s", dbname)
cur.execute('CREATE DATABASE ' + dbname)
database_created = True
if database_created:
with contextlib.closing(psycopg2.connect('postgresql:///' + dbname)) as conn:
conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
with contextlib.closing(conn.cursor()) as cur:
logger.info("create extension postgis")
cur.execute('CREATE EXTENSION postgis;')
code = pathlib.Path('bots/osm')
data = pathlib.Path('data/osm')
output = data / 'LT.tar.gz2'
source = 'http://download.gisgraphy.com/openstreetmap/pbf/LT.tar.bz2'
logger.info('downloading %s', source)
http_code = subprocess.run(funcy.flatten([
'curl', source,
['--time-cond', str(output)] if output.exists() else [],
'--output', str(output),
'--location',
'--silent',
'--write-out', '%{http_code}',
]), check=True, stdout=subprocess.PIPE).stdout
http_code = http_code.decode()
logger.info('http code: %s', http_code)
if http_code == '200':
logger.info('extracting %s', output)
subprocess.run(['tar', '--directory', str(output.parent), '-xjf', str(output)], check=True)
logger.info('importing %s', data / 'LT')
# https://github.com/openstreetmap/osm2pgsql#usage
subprocess.run([
'osm2pgsql',
'--create',
'--database', dbname,
'--style', str(code / 'lietuva.style'),
'--input-reader', 'pbf',
str(data / 'LT'),
], check=True)
logger.info('done')
def get_config():
with closing(open("configuration.json")) as f:
data = f.read()
config = json.loads(data)
return config
def __contains__(self, item):
with locking("%s.lock" % self.metadata_file, LOCK_SH, timeout=TIMEOUT):
with closing(shelve.open(self.metadata_file)) as database:
result = item in database # pylint: disable=E1135
return result
def main():
options = check_environment()
host_list = None
vpc = None # Set if the test owns the VPC
if options.host_list is None:
log.info('CCM_VPC_HOSTS not provided, requesting new VPC from CCM...')
vpc = make_vpc(use_bare_os=options.test_install_prereqs)
host_list = vpc.hosts()
else:
host_list = options.host_list
assert os.path.exists(
'ssh_key'), 'Valid SSH key for hosts must be in working dir!'
# key must be chmod 600 for test_runner to use
os.chmod('ssh_key', stat.S_IREAD | stat.S_IWRITE)
# Create custom SSH Runnner to help orchestrate the test
ssh_user = '******'
ssh_key_path = 'ssh_key'
remote_dir = '/home/centos'
if options.use_api:
installer = test_util.installer_api_test.DcosApiInstaller()
if not options.test_install_prereqs:
# If we dont want to test the prereq install, use offline mode to avoid it
installer.offline_mode = True
else:
installer = test_util.installer_api_test.DcosCliInstaller()
host_list_w_port = [i + ':22' for i in host_list]
@retry(stop_max_delay=120000)
def establish_host_connectivity():
"""Continually try to recreate the SSH Tunnels to all hosts for 2 minutes
"""
return closing(
TunnelCollection(ssh_user, ssh_key_path, host_list_w_port))
log.info('Checking that hosts are accessible')
with establish_host_connectivity() as tunnels:
local_ip = {}
for tunnel in tunnels.tunnels:
local_ip[tunnel.host] = get_local_address(tunnel, remote_dir)
if options.do_setup:
# Make the default user priveleged to use docker
tunnel.remote_cmd(
['sudo', 'usermod', '-aG', 'docker', ssh_user])
# use first node as bootstrap node, second node as master, all others as agents
test_host = host_list[0]
test_host_local = local_ip[host_list[0]]
master_list = [local_ip[host_list[1]]]
agent1 = local_ip[host_list[2]]
agent2 = local_ip[host_list[3]]
agent_list = [agent1, agent2]
public_agent_list = [local_ip[host_list[4]]]
log.info('Test host public/private IP: ' + test_host + '/' +
test_host_local)
with closing(SSHTunnel(ssh_user, ssh_key_path,
test_host)) as test_host_tunnel:
log.info('Setting up installer on test host')
installer.setup_remote(tunnel=test_host_tunnel,
installer_path=remote_dir +
'/dcos_generate_config.sh',
download_url=options.installer_url)
if options.do_setup:
# only do on setup so you can rerun this test against a living installer
log.info('Verifying installer password hashing')
test_pass = '******'
hash_passwd = installer.get_hashed_password(test_pass)
assert passlib.hash.sha512_crypt.verify(
test_pass, hash_passwd), 'Hash does not match password'
if options.use_api:
installer.start_web_server()
with open(pkg_resources.resource_filename(
"gen", "ip-detect/aws.sh")) as ip_detect_fh:
ip_detect_script = ip_detect_fh.read()
with open('ssh_key', 'r') as key_fh:
ssh_key = key_fh.read()
# Using static exhibitor is the only option in the GUI installer
if options.use_api:
log.info(
'Installer API is selected, so configure for static backend')
zk_host = None # causes genconf to use static exhibitor backend
else:
log.info('Installer CLI is selected, so configure for ZK backend')
zk_host = test_host_local + ':2181'
zk_cmd = [
'sudo', 'docker', 'run', '-d', '-p', '2181:2181', '-p',
'2888:2888', '-p', '3888:3888', 'jplock/zookeeper'
]
test_host_tunnel.remote_cmd(zk_cmd)
log.info("Configuring install...")
installer.genconf(zk_host=zk_host,
master_list=master_list,
agent_list=agent_list,
public_agent_list=public_agent_list,
ip_detect_script=ip_detect_script,
ssh_user=ssh_user,
ssh_key=ssh_key,
add_config_path=options.add_config_path,
rexray_config_preset='aws')
log.info("Running Preflight...")
if options.test_install_prereqs:
# Runs preflight in --web or --install-prereqs for CLI
# This may take up 15 minutes...
installer.install_prereqs()
if options.test_install_prereqs_only:
if vpc:
vpc.delete()
sys.exit(0)
else:
# Will not fix errors detected in preflight
installer.preflight()
log.info("Running Deploy...")
installer.deploy()
log.info("Running Postflight")
installer.postflight()
with closing(SSHTunnel(ssh_user, ssh_key_path,
host_list[1])) as master_tunnel:
# Runs dcos-image integration tests inside the cluster
result = test_util.test_runner.integration_test(
tunnel=master_tunnel,
test_dir=remote_dir,
region=vpc.get_region() if vpc else DEFAULT_AWS_REGION,
dcos_dns=master_list[0],
master_list=master_list,
agent_list=agent_list,
public_agent_list=public_agent_list,
provider='onprem',
# Setting dns_search: mesos not currently supported in API
test_dns_search=not options.use_api,
aws_access_key_id=options.aws_access_key_id,
aws_secret_access_key=options.aws_secret_access_key,
add_env=options.add_env,
pytest_dir=options.pytest_dir,
pytest_cmd=options.pytest_cmd)
if result == 0:
log.info("Test successsful! Deleting VPC if provided in this run...")
if vpc is not None:
vpc.delete()
else:
log.info(
"Test failed! VPC will remain for debugging 1 hour from instantiation"
)
if options.ci_flags:
result = 0 # Wipe the return code so that tests can be muted in CI
sys.exit(result)
def __delitem__(self, key):
with locking("%s.lock" % self.metadata_file, LOCK_EX, timeout=TIMEOUT):
with closing(shelve.open(self.metadata_file)) as database:
del database[key] # pylint: disable=E1138
def items(self):
with locking("%s.lock" % self.metadata_file, LOCK_SH, timeout=TIMEOUT):
with closing(shelve.open(self.metadata_file)) as database:
for k in database.keys(): # pylint: disable=E1101
yield (k, database[k]) # pylint: disable=E1136
def tcp_socket():
import socket
from contextlib import closing
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
yield sock
def pop(self, *args):
with locking("%s.lock" % self.metadata_file, LOCK_EX, timeout=TIMEOUT):
with closing(shelve.open(self.metadata_file)) as database:
result = database.pop(*args) # pylint: disable=E1101
return result
def unused_tcp_port() -> int:
"""Return an unused port."""
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
sock.bind(("", 0))
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
return cast(int, sock.getsockname()[1])
def __getitem__(self, key):
with locking("%s.lock" % self.metadata_file, LOCK_SH, timeout=TIMEOUT):
with closing(shelve.open(self.metadata_file)) as database:
result = database[key] # pylint: disable=E1136
return result
def delete(self, key):
with self.lock:
with closing(shelve.open(self.filename)) as cache:
if key in cache:
del cache[key]
def find_free_port():
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
s.bind(("localhost", 0))
# s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
port = s.getsockname()[1]
return port
def get(self, key):
with closing(shelve.open(self.filename)) as cache:
if key in cache:
return cache.get(key)
def clear(self):
with self.lock:
with closing(shelve.open(self.filename)) as cache:
cache.clear()
def get_free_port(host='127.0.0.1'):
"""Return an unused TCP port."""
with contextlib.closing(socket.socket()) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, 0))
return sock.getsockname()[1]
def set(self, key, value):
with self.lock:
with closing(shelve.open(self.filename)) as cache:
cache.setdefault(key, value)
def copy_to_cache(self, ident):
ident = unquote(ident)
#get source image and write to temporary file
(source_url, options) = self._web_request_url(ident)
assert source_url is not None
cache_dir = self.cache_dir_path(ident)
mkdir_p(cache_dir)
with closing(requests.get(source_url, stream=True, **options)) as response:
if not response.ok:
logger.warn(
"Source image not found at %s for identifier: %s. "
"Status code returned: %s.",
source_url, ident, response.status_code
)
raise ResolverException(
"Source image not found for identifier: %s. "
"Status code returned: %s." % (ident, response.status_code)
)
extension = self.cache_file_extension(ident, response)
local_fp = join(cache_dir, "loris_cache." + extension)
with tempfile.NamedTemporaryFile(dir=cache_dir, delete=False) as tmp_file:
for chunk in response.iter_content(2048):
tmp_file.write(chunk)
# Now rename the temp file to the desired file name if it still
# doesn't exist (another process could have created it).
#
# Note: This is purely an optimisation; if the file springs into
# existence between the existence check and the copy, it will be
# overridden.
if exists(local_fp):
logger.info('Another process downloaded src image %s', local_fp)
remove(tmp_file.name)
else:
safe_rename(tmp_file.name, local_fp)
logger.info("Copied %s to %s", source_url, local_fp)
# Check for rules file associated with image file
# These files are < 2k in size, so fetch in one go.
# Assumes that the rules will be next to the image
# cache_dir is image specific, so this is easy
bits = split(source_url)
fn = bits[1].rsplit('.')[0] + "." + self.auth_rules_ext
rules_url = bits[0] + '/' + fn
try:
resp = requests.get(rules_url)
if resp.status_code == 200:
local_rules_fp = join(cache_dir, "loris_cache." + self.auth_rules_ext)
if not exists(local_rules_fp):
with open(local_rules_fp, 'w') as fh:
fh.write(resp.text)
except:
# No connection available
pass
return local_fp
print("Downloading blender from " + url)
file_tmp = urlretrieve(url, None, DownloadProgressBar())[0]
except ImportError:
print("Progressbar for downloading, can only be shown, "
"when the python package \"progressbar\" is installed")
file_tmp = urlretrieve(url, None)[0]
if platform == "linux" or platform == "linux2":
if version_info.major == 3:
with tarfile.open(file_tmp) as tar:
tar.extractall(blender_install_path)
else:
with contextlib.closing(lzma.LZMAFile(file_tmp)) as xz:
with tarfile.open(fileobj=xz) as f:
f.extractall(blender_install_path)
elif platform == "darwin":
if not os.path.exists(blender_install_path):
os.makedirs(blender_install_path)
os.rename(file_tmp, os.path.join(blender_install_path, blender_version + ".dmg"))
# installing the blender app by mounting it and extracting the information
subprocess.Popen(["hdiutil attach {}".format(os.path.join(blender_install_path, blender_version + ".dmg"))],
shell=True).wait()
subprocess.Popen(
["cp -r {} {}".format(os.path.join("/", "Volumes", "Blender", "Blender.app"), blender_install_path)],
shell=True).wait()
subprocess.Popen(["diskutil unmount {}".format(os.path.join("/", "Volumes", "Blender"))], shell=True)
# removing the downloaded image again
subprocess.Popen(["rm {}".format(os.path.join(blender_install_path, blender_version + ".dmg"))], shell=True).wait()
def get_duration(fname):
with contextlib.closing(wave.open(fname, 'r')) as f:
frames = f.getnframes()
rate = f.getframerate()
duration = frames / float(rate)
return duration
def check_connection_ntuple(conn):
"""Check validity of a connection namedtuple."""
# check ntuple
assert len(conn) in (6, 7), conn
has_pid = len(conn) == 7
has_fd = getattr(conn, 'fd', -1) != -1
assert conn[0] == conn.fd
assert conn[1] == conn.family
assert conn[2] == conn.type
assert conn[3] == conn.laddr
assert conn[4] == conn.raddr
assert conn[5] == conn.status
if has_pid:
assert conn[6] == conn.pid
# check fd
if has_fd:
assert conn.fd >= 0, conn
if hasattr(socket, 'fromfd') and not WINDOWS:
try:
dupsock = socket.fromfd(conn.fd, conn.family, conn.type)
except (socket.error, OSError) as err:
if err.args[0] != errno.EBADF:
raise
else:
with contextlib.closing(dupsock):
assert dupsock.family == conn.family
assert dupsock.type == conn.type
# check family
assert conn.family in (AF_INET, AF_INET6, AF_UNIX), repr(conn.family)
if conn.family in (AF_INET, AF_INET6):
# actually try to bind the local socket; ignore IPv6
# sockets as their address might be represented as
# an IPv4-mapped-address (e.g. "::127.0.0.1")
# and that's rejected by bind()
if conn.family == AF_INET:
s = socket.socket(conn.family, conn.type)
with contextlib.closing(s):
try:
s.bind((conn.laddr[0], 0))
except socket.error as err:
if err.errno != errno.EADDRNOTAVAIL:
raise
elif conn.family == AF_UNIX:
assert conn.status == psutil.CONN_NONE, conn.status
# check type (SOCK_SEQPACKET may happen in case of AF_UNIX socks)
assert conn.type in (SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET), \
repr(conn.type)
if conn.type == SOCK_DGRAM:
assert conn.status == psutil.CONN_NONE, conn.status
# check laddr (IP address and port sanity)
for addr in (conn.laddr, conn.raddr):
if conn.family in (AF_INET, AF_INET6):
assert isinstance(addr, tuple), addr
if not addr:
continue
assert isinstance(addr.port, int), addr.port
assert 0 <= addr.port <= 65535, addr.port
check_net_address(addr.ip, conn.family)
elif conn.family == AF_UNIX:
assert isinstance(addr, str), addr
# check status
assert isinstance(conn.status, str), conn
valids = [getattr(psutil, x) for x in dir(psutil) if x.startswith('CONN_')]
assert conn.status in valids, conn
def was_spam(self, searchkey):
with closing(bsddb.btopen(self.spamdb, 'c')) as db:
return searchkey in db
def wrapper(self, *args, **kwargs):
with contextlib.closing(self.mysql_connect()) as connection:
with contextlib.closing(connection.cursor()) as cursor:
return func(self, cursor, *args, **kwargs)
def mark_spam(self, searchkey):
with closing(bsddb.btopen(self.spamdb, 'c')) as db:
db[searchkey] = ''
async def handle_report(self, data):
with closing(self.db.cursor()) as cursor:
cursor.execute(
'''
-- Find tasks with a matching outhash (that is, tasks that
-- are equivalent)
SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND outhash=:outhash
-- If there is an exact match on the taskhash, return it.
-- Otherwise return the oldest matching outhash of any
-- taskhash
ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END,
created ASC
-- Only return one row
LIMIT 1
''', {k: data[k]
for k in ('method', 'outhash', 'taskhash')})
row = cursor.fetchone()
# If no matching outhash was found, or one *was* found but it
# wasn't an exact match on the taskhash, a new entry for this
# taskhash should be added
if row is None or row['taskhash'] != data['taskhash']:
# If a row matching the outhash was found, the unihash for
# the new taskhash should be the same as that one.
# Otherwise the caller provided unihash is used.
unihash = data['unihash']
if row is not None:
unihash = row['unihash']
insert_data = {
'method': data['method'],
'outhash': data['outhash'],
'taskhash': data['taskhash'],
'unihash': unihash,
'created': datetime.now()
}
for k in ('owner', 'PN', 'PV', 'PR', 'task',
'outhash_siginfo'):
if k in data:
insert_data[k] = data[k]
cursor.execute(
'''INSERT INTO tasks_v2 (%s) VALUES (%s)''' %
(', '.join(sorted(insert_data.keys())), ', '.join(
':' + k for k in sorted(insert_data.keys()))),
insert_data)
self.db.commit()
logger.info('Adding taskhash %s with unihash %s',
data['taskhash'], unihash)
d = {
'taskhash': data['taskhash'],
'method': data['method'],
'unihash': unihash
}
else:
d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
self.write_message(d)
