def sqlize(rdf_file, kb_name, binary=True):
"""Convert an RDF file into a SQL dump.
Arguments:
rdf_file (str): Path to the input RDF file.
kb_name (str): The name of the KB. Used as the stem for the output filename.
binary (bool): Whether to save to a binary SQLite file. Defaults to True.
Returns:
str: Filename of the output file.
Raises:
FileExistsError: If any intermediate files already exist.
"""
rdf_file = realpath(expanduser(rdf_file))
directory = dirname(rdf_file)
sql_file = join_path(directory, kb_name + '.sql')
if file_exists(sql_file):
raise FileExistsError(sql_file)
RDFSQLizer().sqlize(rdf_file, kb_name, sql_file)
if binary:
rdfsqlite_file = join_path(directory, kb_name + '.rdfsqlite')
if file_exists(rdfsqlite_file):
raise FileExistsError(sql_file)
read_dump(sql_file, rdfsqlite_file)
remove(sql_file)
return rdfsqlite_file
else:
return sql_file
def cli_args_helper(arguments, parser):
from os.path import exists as file_exists
if (not (arguments.help)):
if (not (arguments.sample)):
exiting("The path to the sample file is empty!", -1)
else:
if (not (file_exists(arguments.sample))):
exiting("The file < " + arguments.sample + " > doesn't exist!",
-1)
if (not (arguments.out_file)):
exiting("The path to the payload output file is empty!", -1)
else:
if (file_exists(arguments.out_file)):
exiting(
"The file < " + arguments.out_file + " > already exist!",
-1)
if (arguments.out_stage1):
if (file_exists(arguments.out_stage1)):
exiting(
"The file <" + arguments.out_stage1 + "> already exist!",
-1)
# Help message and exit.
if ((arguments.help) or (len(argv) <= 1)):
parser.print_help()
exit(0)
return (arguments)
def test_episodes_and_examples_to_json(self):
episode_1 = Episode(('a', 'b'))
episode_2 = Episode(('b', ))
examples_1 = EventSequences(event_sequences=(EventSequence(
sequence_of_events=[Event('a', 1), Event('b', 2)]
), EventSequence(
sequence_of_events=[Event('a', 2), Event('b', 3)])))
examples_2 = EventSequences(
event_sequences=(EventSequence(sequence_of_events=[Event('b', 2)]),
EventSequence(
sequence_of_events=[Event('b', 3)])))
episode_1.examples = examples_1
episode_2.examples = examples_2
episodes = Episodes((episode_1, episode_2))
episodes.to_json('episodes.txt')
self.assertTrue(file_exists('episodes.txt'))
# TODO: test 'episodes.txt' contents
file_remove('episodes.txt')
episodes.examples_to_json('examples.txt')
self.assertTrue(file_exists('examples.txt'))
with open('examples.txt') as f:
result = f.read()
expected = '''[[["a", 1], ["b", 2]], [["a", 2], ["b", 3]]]\n[[["b", 2]], [["b", 3]]]\n'''
self.assertEqual(expected, result)
file_remove('examples.txt')
result = episodes.examples_to_json()
self.assertEqual(expected, result)
def test_restore_with_relative_path(self):
self.fake_trash_dir.add_trashed_file(
"file1", pj(self.curdir, "path", "to", "file1"), "contents")
self.assertEqual(
True, file_exists(pj(self.trash_dir, "info", "file1.trashinfo")))
self.assertEqual(True, file_exists(pj(self.trash_dir, "files",
"file1")))
result = self.run_command(
"trash-restore", ["%(curdir)s" % {
'curdir': "."
}, '--sort=path'],
input='0')
self.assertEqual(
"""\
0 2000-01-01 00:00:01 %(curdir)s/path/to/file1
What file to restore [0..0]: """ % {'curdir': self.curdir}, result.stdout)
self.assertEqual("", result.stderr)
self.assertEqual("contents", read_file(pj(self.curdir,
"path/to/file1")))
self.assertEqual(
False, file_exists(pj(self.trash_dir, "info", "file1.trashinfo")))
self.assertEqual(False,
file_exists(pj(self.trash_dir, "files", "file1")))
def get_sales_train_val(self, memopt=True) -> pd.DataFrame:
"""
Get sales_train_validation.csv dataframe
Args:
memopt (bool, optional): do memory optimization. Defaults to True.
Returns:
pd.DataFrame: sales train validation dataframe
"""
save_path = self.data_root + "/processed/stv_memopt.bin"
read_file = "/extracted/sales_train_validation.csv"
if file_exists(save_path):
print(f"Loading {save_path}")
df = pd.read_pickle(save_path)
else:
df = pd.read_csv(self.data_root + read_file)
if memopt:
df = self.util_handle.reduce_memory_usage(df)
if not file_exists(save_path):
print(f"Saving {save_path}")
df.to_pickle(save_path, )
else:
pass
return df
def test_episodes_and_examples_to_json(self):
episode_1 = Episode(('a', 'b'))
episode_2 = Episode(('b',))
examples_1 = EventSequences(event_sequences=(EventSequence(sequence_of_events=[Event('a', 1), Event('b', 2)]),
EventSequence(sequence_of_events=[Event('a', 2), Event('b', 3)])))
examples_2 = EventSequences(event_sequences=(EventSequence(sequence_of_events=[Event('b', 2)]),
EventSequence(sequence_of_events=[Event('b', 3)])))
episode_1.examples = examples_1
episode_2.examples = examples_2
episodes = Episodes((episode_1, episode_2))
episodes.to_json('episodes.txt')
self.assertTrue(file_exists('episodes.txt'))
# TODO: test 'episodes.txt' contents
file_remove('episodes.txt')
episodes.examples_to_json('examples.txt')
self.assertTrue(file_exists('examples.txt'))
with open('examples.txt') as f:
result = f.read()
expected='''[[["a", 1], ["b", 2]], [["a", 2], ["b", 3]]]\n[[["b", 2]], [["b", 3]]]\n'''
self.assertEqual(expected, result)
file_remove('examples.txt')
result = episodes.examples_to_json()
self.assertEqual(expected, result)
def read_catalog(filebase=None, return_dtype=np.float):
"""
Reads a galaxy/randoms catalog and returns 3 XYZ arrays.
Parameters
-----------
filebase: string (optional)
The fully qualified path to the file. If omitted, reads the
theory galaxy catalog under ../theory/tests/data/
return_dtype: numpy dtype for returned arrays. Default ``numpy.float``
Specifies the datatype for the returned arrays. Must be in
{np.float, np.float32}
Returns
--------
``x y z`` - Unpacked numpy arrays compatible with the installed
version of ``Corrfunc``.
.. note:: If the filename is omitted, then first the fast-food file
is searched for, and then the ascii file. End-users should always
supply the full filename.
"""
if filebase is None:
filename = pjoin(dirname(abspath(__file__)), "../theory/tests/data/",
"gals_Mr19")
allowed_exts = {
'.ff': read_fastfood_catalog,
'.txt': read_ascii_catalog,
'.dat': read_ascii_catalog,
'.csv': read_ascii_catalog
}
for e in allowed_exts:
if file_exists(filename + e):
f = allowed_exts[e]
x, y, z = f(filename + e, return_dtype)
return x, y, z
raise IOError("Could not locate {0} with any of these extensions \
= {1}".format(filename, allowed_exts.keys()))
else:
# Likely an user-supplied value
if file_exists(filebase):
extension = splitext(filebase)[1]
f = read_fastfood_catalog if '.ff' in extension else read_ascii_catalog
# default return is double
x, y, z = f(filebase, return_dtype)
return x, y, z
raise IOError("Could not locate file {0}".format(filebase))
def main(command, arg):
if command == 'list':
history = HistoryHandler()
history.apply_file(arg)
elif command == 'download':
if len(glob(arg + ".*")) == 0:
CatChangeset.download(int(arg))
sleep(apidelay)
elif command == 'process':
fn = arg.replace('.osm', '.osc')
if len(glob(fn + '*')) == 0:
try:
cs = CatChangeset(arg)
log.info(f"{cs.id} has {len(cs.buildings)} buildings and "
f"{len(cs.parts)} parts")
cs.get_missing_parts()
if cs.error > 0 and not DEBUG:
log.error(f"{cs.id} has errors")
elif len(cs.osc.ways) + len(cs.osc.relations) > 0:
cs.osc.write(DEBUG)
else:
log.warning(f"{cs.id} has no missing building parts")
except RuntimeError:
if file_exists(fn):
os.remove(fn)
log.error(f"{arg.replace('.osm', '')} runtime error")
if file_exists(arg):
os.remove(arg)
elif command == 'upload':
if DEBUG:
print("This option is intentionally deactivated")
elif not file_exists(arg + '.gz'):
csid = arg.replace('.osc', '')
upload = UploadHandler()
upload.apply_file(arg)
cs = api.ChangesetCreate({
'comment': cscomment + csid,
'source': sourcetext,
'type': 'bot',
'url': csurl,
})
try:
api.ChangesetUpload(upload.data)
api.ChangesetClose()
log.info(f"{csid} fixed in changeset {cs}")
with open(arg, 'rb') as f_in:
with gzip.open(arg + '.gz', 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
os.remove(arg)
except osmapi.ApiError as e:
log.error(f"{csid} {str(e)}")
os.rename(arg, arg + '.failed')
sleep(apidelay)
else:
help()
def test_dot_dot_argument_is_skipped_even_in_subdirs(self):
self.run_trashput("trash-put", "sandbox/..", "other_argument")
# the dot directory shouldn't be operated, but a diagnostic message
# shall be writtend on stderr
self.stderr_should_be(
"trash-put: cannot trash '..' directory 'sandbox/..'\n")
# the remaining arguments should be processed
assert not file_exists('other_argument')
assert file_exists('sandbox')
def lookup(self):
filename = self.template.filename
path = '%%s/%s'%filename
file = None
if file_exists(filename) and isfile(filename):
file = open(filename)
else:
for dir in self.template.directories:
if file_exists(path%dir) and isfile(path%dir):
file = open(path%dir)
break
return file
def __init__(self, username):
user_file = open(
'Save Location.txt').read() + 'UserData/' + username.lower()
history_file = readCSVFile(user_file + '/history.csv') if file_exists(
user_file + '/history.csv') else []
datafile = readCSVFile(user_file + '/data.csv') if file_exists(
user_file + '/data.csv') else []
self.username = username
self.historyDict = history_file
self.historyPath = 'UserData/' + self.username + '/GamesHistory/GamesHistory.csv'
data = None if len(datafile) < 1 else datafile[0]
self.firstName = data['FirstName']
self.lastName = data['LastName']
self.registered = self.username in GetRegisteredUsers()
del data
def try_load_or_create_new(session_user_id):
"""Loads a saved session_user_id.session or creates a new one.
If session_user_id=None, later .save()'s will have no effect.
"""
if session_user_id is None:
return Session(None)
else:
path = '{}.session'.format(session_user_id)
result = Session(session_user_id)
if not file_exists(path):
return result
try:
with open(path, 'r') as file:
data = json.load(file)
result.port = data.get('port', result.port)
result.salt = data.get('salt', result.salt)
result.layer = data.get('layer', result.layer)
result.server_address = \
data.get('server_address', result.server_address)
# FIXME We need to import the AuthKey here or otherwise
# we get cyclic dependencies.
from ..crypto import AuthKey
if data.get('auth_key_data', None) is not None:
key = b64decode(data['auth_key_data'])
result.auth_key = AuthKey(data=key)
for e_mid, e_hash in data.get('entities', []):
result._input_entities[e_mid] = e_hash
except (json.decoder.JSONDecodeError, UnicodeDecodeError):
pass
return result
def cache_calc(filename, func, *args, **kwargs):
""" Cache calculations, so that the first call to this function performs the
calculations, and caches them to a file. Future calls to this function
simply load up the data from the cached file.
Args:
filename:(str)
the file path you want to save the cached file as
func: (callable)
The function to call to calculate the
*args:
ordered arguments to be passed on to func()
**kwargs:
keyword arguments to be passed on to func()
Returns:
Returns whatever func() returns.
Examples:
cache_calc("myCachedFile", myFunc)
"""
# ==========================================================================
if file_exists(filename):
print("Loading the cached version of " + filename)
with open(filename, mode="rb") as fileObj:
x = pickle_load(fileObj)
else:
print("Caching the calculation to the file " + filename)
x = func(*args, **kwargs)
# Cache the calculation so future calls to this function load the cached
# object instead.
with open(filename, mode="wb") as fileObj:
pickle_dump(x, fileObj)
return x
def test_save_different_tracebacks(self):
path_1 = join(LOG_DIRECTORY, exc_utils.EXCEPTION_FILE_NAME_1)
path_2 = join(LOG_DIRECTORY, exc_utils.EXCEPTION_FILE_NAME_2)
exception_info = exc_utils.get_exception_info_1()
storage = FileStorage(LOG_DIRECTORY)
storage.save("diff:first", exception_info)
self.assertTrue(file_exists(path_1))
exception_info = exc_utils.get_exception_info_2()
storage = FileStorage(LOG_DIRECTORY)
storage.save("diff:second", exception_info)
self.assertTrue(file_exists(path_2))
self.assertEqual("diff:first", self._get_file_contents(path_1))
self.assertEqual("diff:second", self._get_file_contents(path_2))
def PutPQR(molecule, filename, standard=False, title='Created by Python chemistry package'):
""" Writes a PQR based on the PQR version 3 spec """
# define an array with location of TER cards and call the function to initialize it
ter_locations = []
_needTer(molecule, ter_locations)
# Define a counter for which atom we're printing out
atom_counter = 1
# Keep track of whether an atom is ATOM or HETATM so we know what to do for CONECTs
rectypes = []
# Keep counters for MASTER record
numters = 0
numconect = 0
# Make sure we aren't overwriting this file if we haven't said it's OK yet
if file_exists(filename) and not overwrite:
raise(exceptions.FileError('Cannot open %s for writing. It already exists.' % filename))
# Open the file for writing and print out the title
file = open(filename,'w',0)
print >> file, "REMARK %s" % title
print >> file, "REMARK %s" % molecule.title
# Now print out each line for atom
for i in range(len(molecule.atoms)):
record, resname = _resname(molecule.residues[molecule.residue_container[i]], standard)
rectypes.append(record)
line = "%-6s %4d %4s %3s %4i %8.3f %7.3f %7.3f %7.4f %7.4f %2s" % \
(record, atom_counter % 10000, _format(molecule.atoms[i]), resname,
molecule.residue_container[i]+1, molecule.coords[3*i], molecule.coords[3*i+1],
molecule.coords[3*i+2], molecule.charges[i], molecule.radii[i], molecule.elements[i])
print >> file, line
atom_counter += 1
try:
if molecule.residue_container[i] != molecule.residue_container[i+1] and \
molecule.residue_container[i] in ter_locations:
print >> file, 'TER'
numters += 1
except IndexError:
pass
atom_counter -= 1 # decrement atom counter, since it was incremented after the last atom
# Now print out the CONECT cards if they are needed
for i in range(len(rectypes)):
if rectypes[i] == 'HETATM' and molecule.residues[molecule.residue_container[i]] != 'WAT':
line = 'CONECT%5d' % (i + 1)
for j in range(len(molecule.bonds[i])):
line += '%5d' % (molecule.bonds[i][j] + 1)
print >> file, line
numconect += 1
# Now print out the MASTER record
print >> file, 'MASTER 1 0 0 0 0 0 0 0 %4d %4d %4d 0' % \
(atom_counter, numters, numconect)
# End the PQR
print >> file, 'END'
def test_save_different_tracebacks(self):
path_1 = join(LOG_DIRECTORY, exc_utils.EXCEPTION_FILE_NAME_1)
path_2 = join(LOG_DIRECTORY, exc_utils.EXCEPTION_FILE_NAME_2)
exception_info = exc_utils.get_exception_info_1()
storage = FileStorage(LOG_DIRECTORY)
storage.save("diff:first", exception_info)
self.assertTrue(file_exists(path_1))
exception_info = exc_utils.get_exception_info_2()
storage = FileStorage(LOG_DIRECTORY)
storage.save("diff:second", exception_info)
self.assertTrue(file_exists(path_2))
self.assertEqual("diff:first", self._get_file_contents(path_1))
self.assertEqual("diff:second", self._get_file_contents(path_2))
def keys_registration(self):
filepath = control.transPath(
control.join(control.addon('plugin.video.youtube').getAddonInfo('profile'), 'api_keys.json'))
setting = control.addon('plugin.video.youtube').getSetting('youtube.allow.dev.keys') == 'true'
if file_exists(filepath):
f = open(filepath)
jsonstore = json.load(f)
try:
old_key_found = jsonstore['keys']['developer'][control.addonInfo('id')]['api_key'] == 'AIzaSyB99XT3fOBkJrK8HvuXYabZ-OEKiooV34A'
except KeyError:
old_key_found = False
no_keys = control.addonInfo('id') not in jsonstore.get('keys', 'developer').get('developer') or old_key_found
if setting and no_keys:
keys = json.loads(decompress(b64decode(self.scramble)))
register_api_keys(control.addonInfo('id'), keys['api_key'], keys['id'], keys['secret'])
f.close()
def check_inputstream_addon():
try:
addon_enabled = control.addon_details('inputstream.adaptive').get('enabled')
except KeyError:
addon_enabled = False
leia_plus = control.kodi_version() >= 18.0
first_time_file = control.join(control.dataPath, 'first_time')
if not addon_enabled and not file_exists(first_time_file) and leia_plus:
try:
yes = control.yesnoDialog(control.lang(30014))
if yes:
control.enable_addon('inputstream.adaptive')
control.infoDialog(control.lang(30402))
with open(first_time_file, 'a'):
pass
except Exception:
pass
def get_chromedriver_location():
""" Solve chromedriver access issues """
CD = Settings.chromedriver_location
if OS_ENV == "windows":
if not CD.endswith(".exe"):
CD += ".exe"
if not file_exists(CD):
workspace_path = slashen(WORKSPACE["path"], "native")
assets_path = "{}{}assets".format(workspace_path, native_slash)
validate_path(assets_path)
# only import from this package when necessary
from instapy_chromedriver import binary_path
CD = binary_path
chrome_version = pkg_resources.get_distribution(
"instapy_chromedriver").version
message = "Using built in instapy-chromedriver executable (version {})".format(
chrome_version)
highlight_print(Settings.profile["name"], message, "workspace", "info",
Settings.logger)
# save updated path into settings
Settings.chromedriver_location = CD
return CD
def main():
"""Main method executed when run"""
if len(argv) != 2:
print("Usage: flymirror.py [rules_file]")
return
if not file_exists(argv[1]):
print("Error: rules file", argv[1], "does not exist.")
return
config = read_config(argv[1])
URLS.put(config.start)
# Start the loops in another thread
# include 2 extra threads for the loopers
pool = ThreadPoolExecutor(int(config.workers) + 2)
perfprint("[START]")
pool.submit(download_loop, pool)
pool.submit(handle_response_loop, pool, config)
# Join on both the queues at once (Yeah, this is hacky -- may break in later versions)
while URLS.unfinished_tasks or RESPONSES.unfinished_tasks:
sleep(0.3)
# Shut everything down (may take 1 second)
DONE.put(True)
pool.shutdown()
perfprint("[END]")
def test_file_write(self):
# Check that the file exists after writing it
write_text_file(file_name="test", file_format=".txt", file_lines="test string only", verbose=True)
self.assertTrue(file_exists("test.txt"))
# Check if the one string we passed is written correctly
with open("test.txt", "r") as file:
file_data = file.readline().strip()
self.assertEqual("test string only", file_data)
# Pass an empty list and see if it fails as expected
state = write_text_file(file_name="test", file_format=".txt", file_lines=[], verbose=True)
self.assertFalse(state)
# Write one line, but passed as a list
write_text_file(file_name="test", file_format=".txt", file_lines=["One line given"], verbose=True)
with open("test.txt", "r") as file:
file_data = file.readline().strip()
self.assertEqual("One line given", file_data)
# write a list of strings
test_lines = ["hello", "my", "name", "is", "Simon"]
write_text_file(file_name="test", file_format=".txt", file_lines=test_lines, verbose=True)
# Read the lines and compare with the lines written
with open("test.txt", "r") as file:
file_data = file.readlines()
for line_number in range(len(test_lines)):
self.assertEqual(test_lines[line_number], file_data[line_number].strip())
# Clean up after the test
remove_file("test.txt")
def read_last_session(
file: Union[str, Path]) -> Optional[List[List[Dict[str, Any]]]]:
if not file_exists(file):
return None
with open_lz4(file) as fd:
data = json.load(fd)
schema_version = data['version']
if schema_version != ['sessionrestore', 1]:
raise util.UnsupportedSchema(file, schema_version)
windows = []
for window in data['windows']:
tabs = []
for tab in window['tabs']:
# current entry in the tab, others are history
current_entry = tab['entries'][0]
tabs.append({
'index': tab['index'],
'title': current_entry['title'],
'url': current_entry['url'],
'container': tab['userContextId'],
'last-accessed': tab['lastAccessed']
})
windows.append(tabs)
return windows
def file_open(self):
while True:
path = QtGui.QFileDialog.getOpenFileName(
self, 'Open MIDI sound set',
QtGui.QDesktopServices.storageLocation(
QtGui.QDesktopServices.HomeLocation),
'MIDI files (*.mid);;All files (*)')
if not path: return
if not file_exists(str(path)):
QtGui.QMessageBox.warning(
self, 'File does not exists',
'The selected does not exist.\nCheck the file name and path.'
)
else:
try:
res = self.midi_load(path)
if not res:
retry = QtGui.QMessageBox.information(
self, 'No sounds found',
'It looks like the selected file does not contain any sound.\nTry with another file?',
QtGui.QMessageBox.Yes | QtGui.QMessageBox.No)
if retry != QtGui.QMessageBox.Yes: return
break
except:
QtGui.QMessageBox.warning(
self, 'Unexpected error',
'Something is wrong with the selected file...\nTry with another one.'
)
self.setSource(res, path)
def create_conf():
config_f = configparser.ConfigParser(allow_no_value=True)
config_f["autospec"] = {}
for fname, comment in sorted(config_options.items()):
config_f.set("autospec", "# {}".format(comment))
if file_exists(fname):
config_f["autospec"][fname] = "true"
os.remove(fname)
else:
config_f["autospec"][fname] = "false"
# renamed options need special care
if file_exists("skip_test_suite"):
config_f["autospec"]["skip_tests"] = "true"
os.remove("skip_test_suite")
write_config(config_f)
def __init__(self, filepath, new_contents=None):
self._filepath = filepath
# Allow initializing new file from provided contents
if file_exists(filepath):
self._contents = self._read()
elif new_contents:
self._contents = new_contents
def cache_exists(cache_metadata, function_name, *args, **kwargs):
if function_name not in cache_metadata:
return False, None
new_caches_for_function = []
cache_changed = False
for function_cache in cache_metadata[function_name]:
if function_cache['args'] == str(args) and (function_cache['kwargs']
== str(kwargs)):
max_age_days = int(function_cache['max_age_days'])
file_name = join_path(DISK_CACHE_DIR, function_cache['file_name'])
if file_exists(file_name):
if get_age_of_file(
file_name) > max_age_days != UNLIMITED_CACHE_AGE:
os.remove(file_name)
cache_changed = True
else:
function_value = unpickle_big_data(file_name)
return True, function_value
else:
cache_changed = True
else:
new_caches_for_function.append(function_cache)
if cache_changed:
if new_caches_for_function:
cache_metadata[function_name] = new_caches_for_function
else:
cache_metadata.pop(function_name)
write_cache_file(cache_metadata)
return False, None
def cache_calc(filename, func, *args, **kwargs):
"""
Cache calculations, so that the first call to this function performs the
calculations, and caches them to a file. And future calls to this function
simply load up the data from the cached file.
:param filename:(str) the file path you want to save the cached file as
:param func: The function to call to calculate the
:param *args: ordered arguments to be passed on to func()
:param **kwargs: keyword arguments to be passed on to func()
:return: whatever func() returns.
:examples:
cache_calc("myCachedFile", myFunc)
"""
# ==========================================================================
if file_exists(filename):
print("Loading the cached version of " + filename)
with open(filename, mode="rb") as fileObj:
x = pickle_load(fileObj)
else:
print("Caching the calculation to the file " + filename)
x = func(*args, **kwargs)
# Cache the calculation so future calls to this function load the cached
# object instead.
with open(filename, mode="wb") as fileObj:
pickle_dump(x, fileObj)
return x
def get_state_vars():
curl_file = 'curl.sh'
assert file_exists(curl_file), '`{}` file not found'.format(curl_file)
with open(curl_file) as fd:
data = dict(arg.split('=', maxsplit=1) for arg in re.search("--data '([^']*)'", fd.read()).group(1).split('&'))
assert data, 'could not parse curl arguments'
return unquote(data['__VIEWSTATE'].strip()), unquote(data['__EVENTVALIDATION'].strip())
def delete_old_disk_caches():
cache_metadata = load_cache_metadata_json()
new_cache_metadata = deepcopy(cache_metadata)
cache_changed = False
for function_name, function_caches in cache_metadata.items():
if function_name == _TOTAL_NUMCACHE_KEY:
continue
to_keep = []
for function_cache in function_caches:
max_age_days = int(function_cache['max_age_days'])
file_name = join_path(DISK_CACHE_DIR, function_cache['file_name'])
if not file_exists(file_name):
cache_changed = True
continue
if not get_age_of_file(
file_name) > max_age_days != UNLIMITED_CACHE_AGE:
to_keep.append(function_cache)
continue
logger.info('Removing stale cache file %s, > %d days', file_name,
max_age_days)
cache_changed = True
os.remove(file_name)
if to_keep:
new_cache_metadata[function_name] = to_keep
if cache_changed:
write_cache_file(new_cache_metadata)
def keys_registration():
filepath = control.transPath(
control.join(
control.addon('plugin.video.youtube').getAddonInfo('profile'),
'api_keys.json'))
setting = control.addon('plugin.video.youtube').getSetting(
'youtube.allow.dev.keys') == 'true'
if file_exists(filepath):
f = open(filepath)
jsonstore = json.load(f)
try:
old_key_found = jsonstore['keys']['developer'][control.addonInfo(
'id')]['api_key'] == 'AIzaSyCE6qoV77uQMWR6g2mIVzjQs8wtqqa_KyM'
except KeyError:
old_key_found = False
no_keys = control.addonInfo('id') not in jsonstore.get(
'keys', 'developer').get('developer') or old_key_found
if setting and no_keys:
keys = json.loads(decompress(b64decode(scramble)))
register_api_keys(control.addonInfo('id'), keys['api_key'],
keys['id'], keys['secret'])
control.sleep(200)
f.close()
def is_valid_profile(path: Union[str, Path]) -> bool:
# checking if any of the files does not exist
for file in [PLACES, COOKIES, EXTENSIONS]:
if not file_exists(join_path(path, file)):
return False
return True
def read_bookmarks(file: Union[str, Path]) -> Optional[Bookmark]:
if not file_exists(file):
return None
with open(file) as fd:
data = json.load(fd)
schema_version = data['version']
if schema_version != 1:
raise util.UnsupportedSchema(file, schema_version)
def recursive(bookmark): # type: ignore
title = bookmark['name']
date_added = dt_from_webkit_epoch(bookmark['date_added'])
if bookmark['type'] == 'folder':
return Bookmark.new_folder(
title,
date_added, [recursive(i) for i in bookmark['children']],
date_modified=dt_from_webkit_epoch(bookmark['date_modified']))
return Bookmark.new(bookmark['url'], title, date_added)
roots = data['roots']
toolbar = recursive(roots['bookmark_bar'])
other = recursive(roots['other'])
synced = recursive(roots['synced'])
# NOTE when changing keep the order in sync with firefox/reader.py
return Bookmark.new_folder('root', datetime.datetime.now(),
[toolbar, other, synced])
def dev():
# Allow this to persist, since we aren't as rigorous about keeping state clean
if not file_exists('.denv'):
local('virtualenv .denv')
with virtualenv(DEV_ENV_DIR):
local('pip install -r requirements.txt')
def __init__(self, model, device, max_dist=0.2, max_iou_distance=0.7, max_age=70, n_init=3, nn_budget=100):
# models trained on: market1501, dukemtmcreid and msmt17
if is_model_in_factory(model):
# download the model
model_path = join('deep_sort/deep/checkpoint', model + '.pth')
if not file_exists(model_path):
gdown.download(get_model_link(model), model_path, quiet=False)
self.extractor = FeatureExtractor(
# get rid of dataset information DeepSort model name
model_name=model.rsplit('_', 1)[:-1][0],
model_path=model_path,
device=str(device)
)
else:
if is_model_type_in_model_path(model):
model_name = get_model_type(model)
self.extractor = FeatureExtractor(
model_name=model_name,
model_path=model,
device=str(device)
)
else:
print('Cannot infere model name from provided DeepSort path, should be one of the following:')
show_supported_models()
exit()
self.max_dist = max_dist
metric = NearestNeighborDistanceMetric(
"cosine", self.max_dist, nn_budget)
self.tracker = Tracker(
metric, max_iou_distance=max_iou_distance, max_age=max_age, n_init=n_init)
def load_VIS_LSTM(model_save_file=model_save_file):
if file_exists(model_save_file):
print("Model already exists. Loading...")
vis_lstm_model = load_model(model_save_file)
else:
vis_lstm_model = VIS_LSTM()
return vis_lstm_model
def data_preprocess():
"""Data preprocess function
for data preprocessing and caching"""
if file_exists(data_preprocess_file):
with open(data_preprocess_file, 'rb+') as f:
data_tup = pickle.load(f)
else:
## Extract the sparse matrix data
f = h5py.File('hidden_oxford_mscoco.h5', 'r', encoding='bytes')
data = f['hidden7_data'][:]
shape = f['hidden7_shape'][:]
indices = f['hidden7_indices'][:]
ptr = f['hidden7_indptr'][:]
## Create the matrix by passing the data
mat = sparse.csr_matrix((data, indices, ptr), shape=shape)
img_array = mat.toarray()
## Load the dictionary from the vocabulary
vocab_dict = np.load('cocoqa/' + 'vocab-dict.npy', encoding='bytes')
quest_dict = vocab_dict[0]
quest_word_arr = vocab_dict[1]
ans_dict = vocab_dict[2]
ans_word_arr = vocab_dict[3]
## Create the tuple
data_tup = (img_array, quest_dict, quest_word_arr, ans_dict,
ans_word_arr)
with open(data_preprocess_file, 'wb+') as f:
pickle.dump(data_tup, f)
return data_tup
def env():
if file_exists(VENV_DIR):
local('rm -rf {env}'.format(env=VENV_DIR))
local('python -m virtualenv {env}'.format(env=VENV_DIR))
with virtualenv(VENV_DIR):
local('python -m pip install --upgrade setuptools')
local('python -m pip install -r requirements.txt')
local('python -m pip install pytest')
def programming_exercise_1():
from os.path import exists as file_exists
numbers_file = 'numbers.txt'
if not file_exists(numbers_file):
create_numbers_file(numbers_file)
with open(numbers_file, 'r') as nfh:
for line in nfh:
print(line.rstrip('\n'))
def test_save(self):
exception_info = exc_utils.get_exception_info_1()
storage = FileStorage(LOG_DIRECTORY)
storage.save("test_save", exception_info)
path = join(LOG_DIRECTORY, exc_utils.EXCEPTION_FILE_NAME_1)
self.assertTrue(file_exists(path))
def _import_pkgs(self):
if re_get is not None:
self.requests_installed = True
if StreamData is not None:
self.livestreamer_installed = True
for path in self.possible_paths:
if file_exists(path):
self.livestreamer_path = path
def env():
if file_exists(VENV_DIR):
local('rm -rf {env}'.format(env=VENV_DIR))
local('virtualenv {env}'.format(env=VENV_DIR))
with virtualenv(VENV_DIR):
local('pip install --upgrade setuptools')
local('pip install -r requirements.txt')
local('pip install pytest')
def render_page(renderer, template, **context):
load_data()
for lang, context in data_contexts.items():
outfile = join(SITE_DIR, context['lang_dir'], template.name)
head = dirname(outfile)
if head and not file_exists(head):
makedirs(head)
print(PROMPT_FMT_HTML % (context['lang'], outfile))
template.stream(context).dump(outfile, "utf-8")
def setup_netrc(self, opener):
"""Setup netrc file
:paramt opener: :class:`pycurl.Curl` object
"""
if self._netrc:
opener.setopt(pycurl.NETRC, 1)
if self._netrc_file and file_exists(self._netrc_file):
opener.setopt(pycurl.NETRC_FILE, self._netrc_file)
def _write_pid_file(self, pidfile):
self._create_dir(dirname(pidfile))
if file_exists(pidfile):
raise UnixSetupError('Error - \'{:}\' exists. Process already running ?.'.format(pidfile))
try:
with open(pidfile, 'w') as fd:
fd.write(str(getpid()))
except IOError, e:
raise UnixSetupError('Error - Couldn\'t write pidfile \'{:}\'. Details : {:}.'.format(pidfile, e))
def try_load_or_create_new(session_user_id):
"""Loads a saved session_user_id session, or creates a new one if none existed before.
If the given session_user_id is None, we assume that it is for testing purposes"""
if session_user_id is None:
return Session(None)
else:
path = '{}.session'.format(session_user_id)
if file_exists(path):
with open(path, 'rb') as file:
return pickle.load(file)
else:
return Session(session_user_id)
def load_data(dataFrame="Feat_normalized.csv") :
'''
Load training data from csv file. Load labels from it.
Return matrix, training labels, encoder for labels.
label_encoder uses transforming textual labels to integer and vice versa:
label_encoder.inverse_transform(0) => 'Mammal_melanosome_0'
label_encoder.transform('Mammal_melanosome_0') => 0
'''
label_exists = False
#Tracer()() #TO REMOVE!!!
if type(dataFrame) == type(''):
if len(dataFrame) > 120 or not file_exists(dataFrame):
#If it a string in format of csv and not a filename
dataFrame = StringIO(dataFrame)
#Load file
df = pd.read_csv(dataFrame, delimiter='\t', header=0)
try:
df.set_index(keys = ['accession', 'classname'], inplace=True)
#df = pd.read_csv(dataFrame, delimiter='\t', header=0, index_col=['accession', 'classname'])
label_exists = True
#When not labeled
except KeyError:
print('Features files does not contains labels')
#df = pd.read_csv(dataFrame, delimiter='\t', header=0, index_col='accession')
df.set_index(keys = 'accession', inplace=True)
else:
df = dataFrame
features = df.values
# M: creates numpy array
feature_names=df.columns.values
print("%s features" % (len(feature_names)))
# print("feature_names: %s" %(feature_names))
if label_exists:
# create an object of scikit label encoder that transforms strings to ints
label_encoder = LabelEncoder()
# M: check if works with multiindex
# M: take only label, not protein name
s = df.index.get_level_values('classname')
#print(s.value_counts)
labels = label_encoder.fit_transform((df.index.get_level_values('classname').values))
#print ("labels: %s %s" %(type(labels),labels))
print("labels List: ",list(label_encoder.classes_))
# M: creates numpy matrix (or array)
return (features, labels, label_encoder, feature_names)
accessions = df.index.get_level_values('accession')
#To change the order!! (and in the calling functions)
return features, accessions, feature_names
def do_GET(s): # my function. reem
"""Respond to a GET request."""
img_string = s.path[1:]
# If someone went to "http://something.somewhere.net/foo/bar/",
# then s.path equals "/foo/bar/".
if img_string: # if not empty string
img_path = to_real_path(img_string)
if file_exists(img_path):
MyHandler.send_file(s, img_path)
return
# reached here? there is a problem with the file
MyHandler.send_nothing(s)
def test_calling_method_dump_should_pickle_the_index_object(self):
fp = NamedTemporaryFile(delete=False)
fp.close()
self.filename = fp.name
index = Index()
index.add_document('coffee', 'I liked it')
index.add_document('water', 'I need it')
index.dump(self.filename)
self.assertTrue(file_exists(self.filename))
fp = open(self.filename)
retrieved_index = cPickle.load(fp)
self.assertEquals(len(retrieved_index), 2)
self.assertEquals(set(retrieved_index._index.keys()),
set(['i', 'liked', 'need', 'it']))
def _write_pid_file(self, pidfile, user, group):
self._create_dir(dirname(pidfile))
if file_exists(pidfile):
raise UnixSetupError('Error - \'{:}\' exists. Process already running ?.'.format(pidfile))
try:
with open(pidfile, 'w') as fd:
fd.write(u'{:}'.format(getpid()))
chown(pidfile, getpwnam(user).pw_uid, getgrnam(group).gr_gid)
except IOError as e:
raise UnixSetupError('Error - Couldn\'t write pidfile \'{:}\'. Details : {:}.'.format(pidfile, e))
except OSError as e:
raise UnixSetupError('Error - Couldn\'t change permissions for pidfile \'{:}\'. Details : {:}.'.format(pidfile, e))
def test_override_duplicate_traceback(self):
"""Only the freshest traceback is stored."""
path = join(LOG_DIRECTORY, exc_utils.EXCEPTION_FILE_NAME_1)
exception_info = exc_utils.get_exception_info_1()
storage = FileStorage(LOG_DIRECTORY)
storage.save("duplicate:first", exception_info)
self.assertTrue(file_exists(path))
# duplicated exception
exception_info = exc_utils.get_exception_info_1("arg", 25.3, self)
storage = FileStorage(LOG_DIRECTORY)
storage.save("duplicate:second", exception_info)
self.assertEqual("duplicate:second", self._get_file_contents(path))
def GetPDB(filename):
""" Read in a PDB file and return a molecule object """
if not file_exists(filename):
raise(exceptions.FileError('%s does not exist' % filename))
raise(exceptions.FileError("PDB Reading not yet implemented"))
# number of atoms in the system
natom = 0
# Open the file and parse it
file = open(filename,'r')
for line in file:
if line[0:6] != 'ATOM ' or line[0:6] != 'HETATM':
natom += 1
pass
def cli(carto, file=None):
if file:
text = "load {}".format(file)
else:
text = ""
while not text[:4] in ("exit", "quit"):
print_state = True
if text and " " in text:
cmd, args = text.strip().split(" ", 1)
if cmd.isdigit():
if args == '""':
args = ''
carto.expand(int(cmd), args)
else:
if cmd == "load":
if file_exists(args):
with open(args, "r") as fd:
carto.from_dot(fd.read())
print("Cartographer loaded from {}".format(args))
elif cmd == "new":
qid = carto.qid
carto.qid += 1
carto.expand(qid, args)
carto.questions[""] = qid
elif cmd == "save":
with open(args, "w") as fd:
fd.write(carto.to_dot())
print("Cartographer saved to {}".format(args))
print_state = False
else:
print("Command unknown")
print_state = False
else:
print("Command unknown")
print_state = False
if print_state:
if file:
with open(file, "w") as fd:
fd.write(carto.to_dot())
fd.flush()
fsync(fd.fileno())
else:
print("\n".join("{}\t{}".format(qid, node.question) for qid, node in carto.nodes.items() if not node.answer))
print("")
text = input("> ")
def read_config_opts():
global config_opts
if not file_exists("options.conf"):
create_conf()
config_f = configparser.ConfigParser()
config_f.read("options.conf")
if "autospec" not in config_f.sections():
print("Missing autospec section in options.conf")
sys.exit(1)
for key in config_f["autospec"]:
config_opts[key] = config_f["autospec"].getboolean(key)
# Rewrite the configuration file in case of formatting changes since a
# configuration file may exist without any comments (either due to an older
# version of autospec or if it was user-created)
rewrite_config_opts()
def prebuild(build_dir='/tmp/build_spacy'):
if file_exists(build_dir):
shutil.rmtree(build_dir)
os.mkdir(build_dir)
spacy_dir = path.dirname(__file__)
wn_url = 'http://wordnetcode.princeton.edu/3.0/WordNet-3.0.tar.gz'
build_venv = path.join(build_dir, '.env')
with lcd(build_dir):
local('git clone %s .' % spacy_dir)
local('virtualenv ' + build_venv)
with prefix('cd %s && PYTHONPATH=`pwd` && . %s/bin/activate' % (build_dir, build_venv)):
local('pip install cython fabric fabtools pytest')
local('pip install -r requirements.txt')
local('fab clean make')
local('cp -r %s/corpora/en/wordnet corpora/en/' % spacy_dir)
local('cp %s/corpora/en/freqs.txt.gz corpora/en/' % spacy_dir)
local('PYTHONPATH=`pwd` python bin/init_model.py en lang_data corpora spacy/en/data')
local('fab test')
local('python setup.py sdist')
def collect_tp(graph, folder):
tps = []
gap = 1000
min_start = sys.maxint
min_end = sys.maxint
for client in graph['nodes']:
tp_file = folder + 'throughput_client_conservative_overall_' + str(client) + '.log'
if not file_exists(tp_file): continue
f = open(tp_file)
reduce_factor = 1
records = []
start = None
end = None
for line in f:
if line.startswith('#'):
if "(ns)" in line:
reduce_factor = 1000000
elif "(cps)" in line:
reduce_factor = 1000
continue
tmp = [float(field) for field in line.strip().split()]
if start == None: start = tmp[0]
end = tmp[0]
records.append([tmp[0], (tmp[2] / reduce_factor)])
f.close()
if min_start > start: min_start = start
if min_end > end: min_end = end
tps.append(records)
ret = []
t = min_start
while t < min_end:
sum = 0
for records in tps:
for record in records:
if (record[0] <= t + gap):
sum += record[1]
records.remove(record)
else:
break
ret.append([(t - min_start) / 1000, sum])
t += gap
return ret
def try_load_or_create_new(session_user_id):
"""Loads a saved session_user_id.session or creates a new one.
If session_user_id=None, later .save()'s will have no effect.
"""
if session_user_id is None:
return JsonSession(None)
else:
path = '{}.session'.format(session_user_id)
result = JsonSession(session_user_id)
if not file_exists(path):
return result
try:
with open(path, 'r') as file:
data = json.load(file)
result.id = data['id']
result.port = data['port']
result.salt = data['salt']
result._sequence = data['sequence']
result.time_offset = data['time_offset']
result.server_address = data['server_address']
# FIXME We need to import the AuthKey here or otherwise
# we get cyclic dependencies.
from ..crypto import AuthKey
if data['auth_key_data'] is not None:
key = b64decode(data['auth_key_data'])
result.auth_key = AuthKey(data=key)
except (json.decoder.JSONDecodeError, UnicodeDecodeError):
# TODO Backwards-compatibility code
old = Session.try_load_or_create_new(session_user_id)
result.id = old.id
result.port = old.port
result.salt = old.salt
result._sequence = old.sequence
result.time_offset = old.time_offset
result.server_address = old.server_address
result.auth_key = old.auth_key
result.save()
return result
def read_history():
history = set()
if file_exists(SHELL_HISTORY):
for shistory in listdir(SHELL_HISTORY):
if not shistory.endswith('.shistory'):
continue
with open(join_path(SHELL_HISTORY, shistory)) as fd:
for entry in fd.readlines():
# only take COMMAND from 'DATE HOST PWD COMMAND'
entry = entry.strip().split("\t", maxsplit=3)[-1].strip()
for line in entry.split(";"):
for conjunction in line.split("|"):
for command in conjunction.split("&&"):
if not is_weird(command) and len(command.split()) > 0:
history.add(command.strip())
# filter out commands that begin with shell keywords
history = set(command for command in history if command.split()[0] not in KEYWORDS)
# filter out commands that begin with non-alphabet characters
history = set(command for command in history if re.match("[a-z]", command))
return history
def _check_migrate_json(self):
if file_exists(self.filename):
try:
with open(self.filename, encoding='utf-8') as f:
data = json.load(f)
self.delete() # Delete JSON file to create database
self._port = data.get('port', self._port)
self._server_address = \
data.get('server_address', self._server_address)
if data.get('auth_key_data', None) is not None:
key = b64decode(data['auth_key_data'])
self._auth_key = AuthKey(data=key)
rows = []
for p_id, p_hash in data.get('entities', []):
if p_hash is not None:
rows.append((p_id, p_hash, None, None, None))
return rows
except UnicodeDecodeError:
return [] # No entities
