def close(self):
"""Close this file, saving the lockfile over the original.
:note: If this method fails, it will attempt to delete the lockfile.
However, it is not guaranteed to do so (e.g. if a filesystem
becomes suddenly read-only), which will prevent future writes to
this file until the lockfile is removed manually.
:raises OSError: if the original file could not be overwritten. The
lock file is still closed, so further attempts to write to the same
file object will raise ValueError.
"""
if self._closed:
return
os.fsync(self._file.fileno())
self._file.close()
try:
if getattr(os, 'replace', None) is not None:
os.replace(self._lockfilename, self._filename)
else:
if sys.platform != 'win32':
os.rename(self._lockfilename, self._filename)
else:
# Windows versions prior to Vista don't support atomic
# renames
_fancy_rename(self._lockfilename, self._filename)
finally:
self.abort()
def _dump(self):
temp = '%s-%s.tmp' % (uuid.uuid4(), self.name)
with open(temp, 'w', encoding='utf-8') as tmp:
json.dump(self._db.copy(), tmp, ensure_ascii=True, cls=self.encoder, separators=(',', ':'))
# atomically move the file
os.replace(temp, self.name)
def save_json(filename: str, data: Union[List, Dict],
private: bool = False) -> None:
"""Save JSON data to a file.
Returns True on success.
"""
tmp_filename = filename + "__TEMP__"
try:
json_data = json.dumps(data, sort_keys=True, indent=4)
mode = 0o600 if private else 0o644
with open(os.open(tmp_filename, O_WRONLY | O_CREAT | O_TRUNC, mode),
'w', encoding='utf-8') as fdesc:
fdesc.write(json_data)
os.replace(tmp_filename, filename)
except TypeError as error:
_LOGGER.exception('Failed to serialize to JSON: %s',
filename)
raise SerializationError(error)
except OSError as error:
_LOGGER.exception('Saving JSON file failed: %s',
filename)
raise WriteError(error)
finally:
if os.path.exists(tmp_filename):
try:
os.remove(tmp_filename)
except OSError as err:
# If we are cleaning up then something else went wrong, so
# we should suppress likely follow-on errors in the cleanup
_LOGGER.error("JSON replacement cleanup failed: %s", err)
def sanitizeTree(filetree):
chapterNames = {}
for root, dirs, files in os.walk(filetree, False):
for name in files:
if name.startswith('.') or name.lower() == 'thumbs.db':
os.remove(os.path.join(root, name))
else:
splitname = os.path.splitext(name)
slugified = slugify(splitname[0])
while os.path.exists(os.path.join(root, slugified + splitname[1])) and splitname[0].upper()\
!= slugified.upper():
slugified += "A"
newKey = os.path.join(root, slugified + splitname[1])
key = os.path.join(root, name)
if key != newKey:
os.replace(key, newKey)
for name in dirs:
if name.startswith('.'):
os.remove(os.path.join(root, name))
else:
tmpName = name
slugified = slugify(name)
while os.path.exists(os.path.join(root, slugified)) and name.upper() != slugified.upper():
slugified += "A"
chapterNames[slugified] = tmpName
newKey = os.path.join(root, slugified)
key = os.path.join(root, name)
if key != newKey:
os.replace(key, newKey)
return chapterNames
def _write_file_bytes(path, data, mode):
"""
Atomically write `data` to `path` using the file permissions
`stat.S_IMODE(mode)`.
File consumers can rely on seeing valid file contents once they are able to
open the file. This is achieved by performing all relevant operations on a
temporary file followed by a `os.replace()` which, if successful, renames to
the desired path (and overwrites upon conflict) in an atomic operation (on
both, Windows, and Linux).
If acting on the temporary file fails (be it writing, closing, chmodding,
replacing) an attempt is performed to remove the temporary file; and the
original exception is re-raised.
"""
assert isinstance(data, bytes)
basename = os.path.basename(path)
tmpfile_dir = os.path.dirname(os.path.realpath(path))
fd, tmpfile_path = tempfile.mkstemp(prefix=basename, dir=tmpfile_dir)
try:
try:
os.write(fd, data)
finally:
os.close(fd)
os.chmod(tmpfile_path, stat.S_IMODE(mode))
os.replace(tmpfile_path, path)
except Exception:
os.remove(tmpfile_path)
raise
def sync_save_log(self, reason):
started = time.time()
new_file = os.path.join(self.log_dir, f"{int(started)}.pickle")
try:
with open(new_file, 'xb') as f:
pickle.dump(self.event_log, f)
except FileExistsError:
log.exception(
f"unable to dump the log, file {new_file} already exists, "
f"too many updates/sec? current: {self.log_updates}, "
f"threshold: {self.log_save_updates}"
)
return False
# atomically replace `current` symlink
tmplink = os.path.join(self.log_dir, "tmp")
try:
os.remove(tmplink)
except FileNotFoundError:
pass
os.symlink(new_file, tmplink)
os.replace(tmplink, self.log_current)
duration = time.time() - started
log.info(f"log dumped to disk because {reason}, took {duration:.4}s")
return True
def save_json(filename: str, data: Union[List, Dict],
private: bool = False, *,
encoder: Optional[json.JSONEncoder] = None) -> None:
"""Save JSON data to a file.
Returns True on success.
"""
tmp_filename = ""
tmp_path = os.path.split(filename)[0]
try:
json_data = json.dumps(data, sort_keys=True, indent=4, cls=encoder)
# Modern versions of Python tempfile create this file with mode 0o600
with tempfile.NamedTemporaryFile(mode="w", encoding='utf-8',
dir=tmp_path, delete=False) as fdesc:
fdesc.write(json_data)
tmp_filename = fdesc.name
if not private:
os.chmod(tmp_filename, 0o644)
os.replace(tmp_filename, filename)
except TypeError as error:
_LOGGER.exception('Failed to serialize to JSON: %s',
filename)
raise SerializationError(error)
except OSError as error:
_LOGGER.exception('Saving JSON file failed: %s',
filename)
raise WriteError(error)
finally:
if os.path.exists(tmp_filename):
try:
os.remove(tmp_filename)
except OSError as err:
# If we are cleaning up then something else went wrong, so
# we should suppress likely follow-on errors in the cleanup
_LOGGER.error("JSON replacement cleanup failed: %s", err)
def updateSingleDocumentToC(input_file, min_toc_len, verbose=False):
"""Add or update table of contents in specified file. Return 1 if file changed, 0 otherwise."""
if verbose :
print( 'file: {}'.format(input_file))
output_file = input_file + '.tmp'
markdownToclify(
input_file=input_file,
output_file=output_file,
min_toc_len=min_toc_len,
github=True,
back_to_top=False,
nolink=False,
no_toc_header=False,
spacer=False,
placeholder=False,
exclude_h=excludeHeadingsFor(input_file))
# prevent race-condition (Python 3.3):
if sys.version_info >= (3, 3):
os.replace(output_file, input_file)
else:
os.remove(input_file)
os.rename(output_file, input_file)
return 1
def move(cls, item, to_collection, to_href):
os.replace(
path_to_filesystem(item.collection._filesystem_path, item.href),
path_to_filesystem(to_collection._filesystem_path, to_href))
cls._sync_directory(to_collection._filesystem_path)
if item.collection._filesystem_path != to_collection._filesystem_path:
cls._sync_directory(item.collection._filesystem_path)
def _save_file(self, path, config, is_theme):
kind = 'theme' if is_theme else 'configuration'
new_path = path + '_.new'
old_path = path + '_.old'
# Write new data
out_data = json.dumps(config, indent=4, cls=_ConfigEncoder, sort_keys=True)
assert len(out_data) <= self._MAX_CHARS
try:
with open(new_path, 'w', encoding='utf-8') as f:
f.write(out_data)
except OSError as e:
print('Could not write new {} file: {}'.format(kind, e.strerror),
file=sys.stderr)
return False
# Replace old file with the new one
try:
if os.path.exists(path):
os.replace(path, old_path)
os.replace(new_path, path)
if os.path.exists(old_path):
os.remove(old_path)
except OSError as e:
print('An error occurred while replacing old {} file: {}'.format(
kind, e.strerror,
file=sys.stderr))
return False
return True
def replaceFile( output_path, input_path ):
# prevent race-condition (Python 3.3):
if sys.version_info >= (3, 3):
os.replace( output_path, input_path )
else:
os.remove( input_path )
os.rename( output_path, input_path )
def main():
infilename = "program_11_10.cpp"
outfilename = "_program_11_10.cpp";
try:
infile = open(infilename, "r")
except:
print("File " + infilename + " did not open")
infile = open(infilename, "r")
try:
outfile = open(outfilename, "w")
except:
print("File " + outfilename + " did not open")
outfile = open(outfilename, "w")
for line_of_text in infile:
match = re.match(r"^[0-9]*", line_of_text)
line_of_text = line_of_text[:match.start()] + line_of_text[match.end():]
outfile.write(line_of_text)
infile.close()
outfile.close()
if os.path.isfile(infilename):
os.remove(infilename)
os.replace(outfilename, infilename)
print_file_list(".", get_file_list("."))
def download(module: str):
name, address = module.split('/')[1], 'https://github.com/{module}/archive/master.zip'.format(module=module)
print('loading module', name, 'from github:', address)
print('starting download ...')
try:
zip_file = request.urlopen(address)
except urllib.error.HTTPError:
print('\nError: No such GitHub project')
raise SystemExit
os.makedirs(os.path.dirname('AElfi/modules/_temp/download.zip'), exist_ok=True)
temporary_zip = open('AElfi/modules/_temp/download.zip', 'wb')
temporary_zip.write(zip_file.read())
zip_file.close()
temporary_zip.close()
print('download complete ...')
temporary_zip = zipfile.ZipFile('AElfi/modules/_temp/download.zip', 'r')
temporary_zip.extractall('AElfi/modules/_temp/download')
temporary_zip.close()
os.remove('AElfi/modules/_temp/download.zip')
print('extracted zip ...')
shutil.rmtree('AElfi/modules/{}/'.format(name), ignore_errors=True)
location, directories = next(os.walk('AElfi/modules/_temp/download'))[:2]
directory = directories[0]
os.replace(location + '/' + directory, 'AElfi/modules/{}/'.format(name))
print('moved to', 'AElfi/modules/{}/'.format(name), '...')
shutil.rmtree('AElfi/modules/_temp/')
print('cleaning up', '...')
for dirname, dirs, files in os.walk('AElfi/modules/{}/'.format(name)):
for file in files:
os.chmod(dirname + '/' + file, 0o755)
print('finised!')
def fileRead(mydir, commit):
"""Reads files recursively and renames it if need."""
data = defaultdict(list)
try:
for fName in os.listdir(mydir):
pathSrc = os.path.join(mydir, fName)
if os.path.isfile(pathSrc):
"""Shrinks only files over 255-bytes length currenty."""
if (utf8len(fName) > 255):
if '.' in fName:
pieces = fName.split('.')
fExt = "."+ pieces[len(pieces)-1]
newfName = " ".join(pieces[:-1])
else:
newfName = fName
fExt = ''
newfName = fineCut(newfName, 255-utf8len(fExt)) + fExt
pathDst = os.path.join(mydir, newfName)
data[mydir].append(fName)
if commit:
os.replace(pathSrc, pathDst)
else:
for k, v in fileRead(pathSrc, commit).items():
data[k].append(str(v).strip("'[]'"))
except OSError as e:
print(e, file=sys.stderr)
return data
def replace_file(src, dst):
"""
Overwrite a file (in an atomic fashion when possible).
:param src: The pathname of the source file (a string).
:param dst: The pathname of the destination file (a string).
"""
# Try os.replace() which was introduced in Python 3.3
# (this should work on POSIX as well as Windows systems).
try:
os.replace(src, dst)
return
except AttributeError:
pass
# Try os.rename() which is atomic on UNIX but refuses to overwrite existing
# files on Windows.
try:
os.rename(src, dst)
return
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Finally we fall back to the dumb approach required only on Windows.
# See https://bugs.python.org/issue8828 for a long winded discussion.
os.remove(dst)
os.rename(src, dst)
def ini_file_helper_add_or_update_key_value(ini_file_name, ini_key_val_pair):
if not Path(ini_file_name).is_file():
return False
if ini_key_val_pair is None or ini_key_val_pair == "":
return False
ini_key_val_pair_splitted = ini_key_val_pair.split(":", maxsplit=1)
key_to_replace = ini_key_val_pair_splitted[0]
val_to_replace = ini_key_val_pair_splitted[1]
## remove 'old' key
ini_out_file_name = ini_file_name + ".tmpout"
with open(ini_file_name, 'r') as ini_file:
with open(ini_out_file_name, 'w') as out_file:
for ini_file_line in ini_file:
if not re.match("^" + key_to_replace + "=", ini_file_line):
out_file.write(ini_file_line)
## append to end
with open(ini_out_file_name, 'a') as out_file:
print(key_to_replace + "=" + val_to_replace, file=out_file)
os.replace(ini_out_file_name, ini_file_name)
return True
def content_write(file, content, append=False):
"""Write content to file. Content may be str or bytes."""
file = path.expanduser(file)
prepare_folder(path.dirname(file))
original = file
if append:
mode = "a"
else:
mode = "w"
file = file + time.strftime("@%Y-%m-%d_%H%M%S")
if isinstance(content, bytes):
mode += "b"
with io.open(file, mode) as f:
write_big(f, content)
else:
if not isinstance(content, str):
content = pprint.pformat(content)
with io.open(file, mode, encoding="utf-8") as f:
write_big(f, content)
if file != original:
os.replace(file, original)
def save_yaml(fname: str, data: JSON_TYPE) -> None:
"""Save a YAML file."""
yaml = YAML(typ='rt')
yaml.indent(sequence=4, offset=2)
tmp_fname = fname + "__TEMP__"
try:
try:
file_stat = os.stat(fname)
except OSError:
file_stat = stat_result(
(0o644, -1, -1, -1, -1, -1, -1, -1, -1, -1))
with open(os.open(tmp_fname, O_WRONLY | O_CREAT | O_TRUNC,
file_stat.st_mode), 'w', encoding='utf-8') \
as temp_file:
yaml.dump(data, temp_file)
os.replace(tmp_fname, fname)
if hasattr(os, 'chown') and file_stat.st_ctime > -1:
try:
os.chown(fname, file_stat.st_uid, file_stat.st_gid)
except OSError:
pass
except YAMLError as exc:
_LOGGER.error(str(exc))
raise HomeAssistantError(exc)
except OSError as exc:
_LOGGER.exception('Saving YAML file %s failed: %s', fname, exc)
raise WriteError(exc)
finally:
if os.path.exists(tmp_fname):
try:
os.remove(tmp_fname)
except OSError as exc:
# If we are cleaning up then something else went wrong, so
# we should suppress likely follow-on errors in the cleanup
_LOGGER.error("YAML replacement cleanup failed: %s", exc)
def atomicupdate(filepath):
dirname, basename = os.path.split(filepath)
if not basename:
raise ValueError("must name a file")
basename, ext = os.path.splitext(basename)
f = _mktemp(dirname, prefix=basename + "-", suffix=ext, delete=False)
tempfilename = f.name
try:
stat = os.stat(filepath)
except FileNotFoundError:
os.chmod(tempfilename, 0o644)
else:
# Copy user, group and access flags
os.chown(tempfilename, stat.st_uid, stat.st_gid)
os.chmod(tempfilename, stat.st_mode)
try:
yield f
except BaseException:
f.close()
os.remove(tempfilename)
raise
else:
f.close()
try:
os.replace(tempfilename, filepath)
except OSError:
os.remove(tempfilename)
raise
def write_two_phases(filename, data, io):
"""
Writes a file in two phase to the filesystem.
First write the data to a temporary file (in the same directory) and than renames the temporary file. If the
file already exists and its content is equal to the data that must be written no action is taken. This has the
following advantages:
* In case of some write error (e.g. disk full) the original file is kep in tact and no file with partially data
is written.
* Renaming a file is atomic. So, running processes will never read a partially written data.
:param str filename: The name of the file were the data must be stored.
:param str data: The data that must be written.
:param pystratum.style.PyStratumStyle.PyStratumStyle io: The output decorator.
"""
write_flag = True
if os.path.exists(filename):
with open(filename, 'r') as file:
old_data = file.read()
if data == old_data:
write_flag = False
if write_flag:
tmp_filename = filename + '.tmp'
with open(tmp_filename, 'w+') as file:
file.write(data)
os.replace(tmp_filename, filename)
io.text('Wrote: <fso>{0}</fso>'.format(filename))
else:
io.text('File <fso>{0}</fso> is up to date'.format(filename))
def move(cls, item, to_collection, to_href):
if not pathutils.is_safe_filesystem_path_component(to_href):
raise pathutils.UnsafePathError(to_href)
os.replace(
pathutils.path_to_filesystem(
item.collection._filesystem_path, item.href),
pathutils.path_to_filesystem(
to_collection._filesystem_path, to_href))
cls._sync_directory(to_collection._filesystem_path)
if item.collection._filesystem_path != to_collection._filesystem_path:
cls._sync_directory(item.collection._filesystem_path)
# Move the item cache entry
cache_folder = os.path.join(item.collection._filesystem_path,
".Radicale.cache", "item")
to_cache_folder = os.path.join(to_collection._filesystem_path,
".Radicale.cache", "item")
cls._makedirs_synced(to_cache_folder)
try:
os.replace(os.path.join(cache_folder, item.href),
os.path.join(to_cache_folder, to_href))
except FileNotFoundError:
pass
else:
cls._makedirs_synced(to_cache_folder)
if cache_folder != to_cache_folder:
cls._makedirs_synced(cache_folder)
# Track the change
to_collection._update_history_etag(to_href, item)
item.collection._update_history_etag(item.href, None)
to_collection._clean_history()
if item.collection._filesystem_path != to_collection._filesystem_path:
item.collection._clean_history()
def update_torrent(client, torrent, new_data, to_save_customs, to_set_customs):
data_file_path = tools.make_sub_name(torrent.get_path(), ".", ".newdata")
with open(data_file_path, "wb") as data_file:
data_file.write(new_data)
with client_hooks(client, torrent, to_save_customs, to_set_customs):
os.replace(data_file_path, torrent.get_path())
torrent.load_from_data(new_data, torrent.get_path())
def dotranscode(action, args):
if action.action != 'transcode':
print('error: dotranscode got non-transcode action:', action)
return
# TODO: make sure the directory exists
# Instead of writing directly to $destpath, write to $destpath.partial,
# so that if we crash we don't leave partially-encoded files laying around
tmppath = action.destpath + ".partial"
cmd = [opusenc_path, '--quiet', '--bitrate', str(args.bitrate), action.filepath, tmppath]
kwargs = dict(
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
with subprocess.Popen(cmd, **kwargs) as process:
try:
_, stderr = process.communicate()
except:
process.kill()
process.wait()
raise
returncode = process.poll()
if returncode != 0:
print("error: transcode failed:", stderr.decode('utf-8', 'replace').strip())
print("info: failed command:", " ".join(cmd))
return
try:
os.replace(tmppath, action.destpath)
except OSError as e:
print("error: rename failed: %s: %s" % (action.destpath, e))
return
def _write_cache():
# We *could* load the cache here, but calling write_cache indicates an error up the stack
assert _cache is not None
with open('cache/index.json.next', 'w') as fp:
json.dump(_cache, fp, sort_keys=True, indent=0) # No indent to save some space
# Hope that os.replace doesn't corrupt the file *and* the git backup.
os.replace('cache/index.json.next', 'cache/index.json')
def write_string(filename, data):
"""
Write a string to a file.
Overwrite any data in that file.
We use an atomic write practice of creating a temporary file and then
moving that temporary file to the given ``filename``. This prevents race
conditions such as the file being read by another process after it is
opened here but not yet written to.
It also prevents us from creating or truncating a file before we fail to
write data to it because of low disk space.
If no file already exists at ``filename``, the new file is created with
permissions 0o644.
"""
prefix = os.path.basename(filename)
tmp_file_dir = os.path.dirname(os.path.realpath(filename))
fd, temporary_filename = tempfile.mkstemp(prefix=prefix, dir=tmp_file_dir)
try:
permissions = os.stat(filename).st_mode
except FileNotFoundError:
permissions = 0o644
try:
try:
os.write(fd, data.encode())
finally:
os.close(fd)
os.chmod(temporary_filename, stat.S_IMODE(permissions))
os.replace(temporary_filename, filename)
except Exception:
os.remove(temporary_filename)
raise
def save(self):
if self.headers['Content-Length'] == None:
self.my_error(411)
return
length = int(self.headers['Content-Length'])
if self.headers['X-File'] in editable_files:
filename = self.headers['X-File']
else:
self.my_error(403)
return
# we're going to be saving really often, so it's likely
# that if the system crashes it'll be during a write,
# so make sure writes are atomic
temp_filename = join(os.path.dirname(abspath(filename)),
'temp-hilarious-editor-'+create_token()+'.txt~')
with open(temp_filename, 'wt', encoding='utf-8') as f:
f.write(self.rfile.read(length).decode('utf-8'))
os.replace(temp_filename, filename)
if on_save != None:
on_save(filename)
self.send_response(204)
self.send_header('Content-Type', 'text/plain')
self.boilerplate_headers()
self.end_headers()
def write_two_phases(the_filename, the_data):
"""
Writes a file in two phase to the filesystem.
First write the data to a temporary file (in the same directory) and than renames the temporary file. If the
file already exists and its content is equal to the data that must be written no action is taken. This has the
following advantages:
* In case of some write error (e.g. disk full) the original file is kep in tact and no file with partially data
is written.
* Renaming a file is atomic. So, running processes will never read a partially written data.
:param str the_filename: The name of the file were the data must be stored.
:param str the_data: The data that must be written.
"""
write_flag = True
if os.path.exists(the_filename):
with open(the_filename, 'r') as file:
old_data = file.read()
if the_data == old_data:
write_flag = False
if write_flag:
tmp_filename = the_filename + '.tmp'
with open(tmp_filename, 'w+') as file:
file.write(the_data)
os.replace(tmp_filename, the_filename)
print("Wrote: '%s'." % the_filename)
def set_info(fname, info):
"""
Writes the magic to the first line that starts with # validated
or # novalidate. If no such line exists, write to the first line
of the file
"""
with open(fname, "r") as fin, tempfile.NamedTemporaryFile(dir=dirname(fname), mode="w", delete=False) as fout:
found = False
written = False
# search for the line first
for line in fin:
if line.startswith("# validated") or line.startswith("# novalidate"):
found = True
break
fin.seek(0)
# Now rewrite the file
for line in fin:
if not written:
if not found:
fout.write(info.line)
written = True
elif line.startswith("# validated") or line.startswith("# novalidate"):
line = info.line
written = True
fout.write(line)
os.replace(fout.name, fname)
async def _download(self):
if self._is_downloading:
return
self._is_downloading = True
try:
result = await extract_info(self.playlist.loop, self.url, download=True)
unmoved_filename = ytdl.prepare_filename(result)
unmoved_filename = md5sum(unmoved_filename, 8).join('-.').join(unmoved_filename.rsplit('.',1))
self.filename = os.path.join(AUDIO_CACHE_PATH, unmoved_filename)
# Ensure the folder that we're going to move into exists.
directory = os.path.dirname(self.filename)
if not os.path.exists(directory):
os.makedirs(directory)
# Move the temporary file to it's final location.
os.replace(ytdl.prepare_filename(result), self.filename)
# Trigger ready callbacks.
self._for_each_future(lambda future: future.set_result(self))
except Exception as e:
self._for_each_future(lambda future: future.set_exception(e))
finally:
self._is_downloading = False
def write_header(header_text, user_filepath, insert_linenum=0, cut_lines=0):
"""Insert header into `user_filepath` starting at line `insert_linenum`
(zero based). Removes `cut_lines` amount of lines after the header.
`cut_lines` is useful for cases where existing header lines need to be
removed.
"""
if insert_linenum > HEADER_IN_FIRST_N_LINES:
raise ValueError(
(
"Header should not be written at line {}. It should be placed"
" near the top of the file."
).format(insert_linenum)
)
try:
with open(user_filepath, "rt") as user_file:
with tempfile.NamedTemporaryFile(
mode="wt",
delete=False) as outfile:
for i, line in enumerate(user_file):
if i == insert_linenum:
outfile.write(header_text)
if i < insert_linenum or i >= insert_linenum + cut_lines:
outfile.write(line)
os.replace(outfile.name, user_filepath)
except UnicodeDecodeError:
print((
"WARNING: File {} is not standard unicode and has been skipped. It"
" may be a binary."
).format(user_filepath))
def backup_dir(src,bkp):
print (datetime.datetime.now()," Backing up " + src + " to " + bkp)
os.makedirs(bkp, mode=0o700, exist_ok=True)
for root, dirs, files in os.walk(src):
for name in files:
os.replace(os.path.join(root, name),os.path.join(bkp, name))
def get_from_cache(
url,
cache_dir=None,
force_download=False,
proxies=None,
etag_timeout=10,
resume_download=False,
user_agent=None,
local_files_only=False,
) -> Optional[str]:
"""
Given a URL, look for the corresponding file in the local cache.
If it's not there, download it. Then return the path to the cached file.
Return:
None in case of non-recoverable file (non-existent or inaccessible url + no cache on disk).
Local path (string) otherwise
"""
if cache_dir is None:
cache_dir = TRANSFORMERS_CACHE
if isinstance(cache_dir, Path):
cache_dir = str(cache_dir)
os.makedirs(cache_dir, exist_ok=True)
etag = None
if not local_files_only:
try:
response = requests.head(url,
allow_redirects=True,
proxies=proxies,
timeout=etag_timeout)
if response.status_code == 200:
etag = response.headers.get("ETag")
except (EnvironmentError, requests.exceptions.Timeout):
# etag is already None
pass
filename = url_to_filename(url, etag)
# get cache path to put the file
cache_path = os.path.join(cache_dir, filename)
# etag is None = we don't have a connection, or url doesn't exist, or is otherwise inaccessible.
# try to get the last downloaded one
if etag is None:
if os.path.exists(cache_path):
return cache_path
else:
matching_files = [
file
for file in fnmatch.filter(os.listdir(cache_dir), filename +
".*")
if not file.endswith(".json") and not file.endswith(".lock")
]
if len(matching_files) > 0:
return os.path.join(cache_dir, matching_files[-1])
else:
# If files cannot be found and local_files_only=True,
# the models might've been found if local_files_only=False
# Notify the user about that
if local_files_only:
raise ValueError(
"Cannot find the requested files in the cached path and outgoing traffic has been"
" disabled. To enable model look-ups and downloads online, set 'local_files_only'"
" to False.")
return None
# From now on, etag is not None.
if os.path.exists(cache_path) and not force_download:
return cache_path
# Prevent parallel downloads of the same file with a lock.
lock_path = cache_path + ".lock"
with FileLock(lock_path):
# If the download just completed while the lock was activated.
if os.path.exists(cache_path) and not force_download:
# Even if returning early like here, the lock will be released.
return cache_path
if resume_download:
incomplete_path = cache_path + ".incomplete"
@contextmanager
def _resumable_file_manager():
with open(incomplete_path, "a+b") as f:
yield f
temp_file_manager = _resumable_file_manager
if os.path.exists(incomplete_path):
resume_size = os.stat(incomplete_path).st_size
else:
resume_size = 0
else:
temp_file_manager = partial(tempfile.NamedTemporaryFile,
dir=cache_dir,
delete=False)
resume_size = 0
# Download to temporary file, then copy to cache dir once finished.
# Otherwise you get corrupt cache entries if the download gets interrupted.
with temp_file_manager() as temp_file:
logger.info(
"%s not found in cache or force_download set to True, downloading to %s",
url, temp_file.name)
http_get(url,
temp_file,
proxies=proxies,
resume_size=resume_size,
user_agent=user_agent)
logger.info("storing %s in cache at %s", url, cache_path)
os.replace(temp_file.name, cache_path)
logger.info("creating metadata file for %s", cache_path)
meta = {"url": url, "etag": etag}
meta_path = cache_path + ".json"
with open(meta_path, "w") as meta_file:
json.dump(meta, meta_file)
return cache_path
def cached_path(
url_or_filename: Union[str, PathLike],
cache_dir: Union[str, Path] = None,
extract_archive: bool = False,
force_extract: bool = False,
) -> str:
"""
Given something that might be a URL (or might be a local path),
determine which. If it's a URL, download the file and cache it, and
return the path to the cached file. If it's already a local path,
make sure the file exists and then return the path.
# Parameters
url_or_filename : `Union[str, Path]`
A URL or local file to parse and possibly download.
cache_dir : `Union[str, Path]`, optional (default = `None`)
The directory to cache downloads.
extract_archive : `bool`, optional (default = `False`)
If `True`, then zip or tar.gz archives will be automatically extracted.
In which case the directory is returned.
force_extract : `bool`, optional (default = `False`)
If `True` and the file is an archive file, it will be extracted regardless
of whether or not the extracted directory already exists.
"""
if cache_dir is None:
cache_dir = CACHE_DIRECTORY
if isinstance(url_or_filename, PathLike):
url_or_filename = str(url_or_filename)
file_path: str
# If we're using the /a/b/foo.zip!c/d/file.txt syntax, handle it here.
exclamation_index = url_or_filename.find("!")
if extract_archive and exclamation_index >= 0:
archive_path = url_or_filename[:exclamation_index]
file_name = url_or_filename[exclamation_index + 1 :]
# Call 'cached_path' recursively now to get the local path to the archive itself.
cached_archive_path = cached_path(archive_path, cache_dir, True, force_extract)
if not os.path.isdir(cached_archive_path):
raise ValueError(
f"{url_or_filename} uses the ! syntax, but does not specify an archive file."
)
# Now return the full path to the desired file within the extracted archive,
# provided it exists.
file_path = os.path.join(cached_archive_path, file_name)
if not os.path.exists(file_path):
raise FileNotFoundError(f"file {file_name} not found within {archive_path}")
return file_path
url_or_filename = os.path.expanduser(url_or_filename)
parsed = urlparse(url_or_filename)
extraction_path: Optional[str] = None
if parsed.scheme in ("http", "https", "s3"):
# URL, so get it from the cache (downloading if necessary)
file_path = get_from_cache(url_or_filename, cache_dir)
if extract_archive and (is_zipfile(file_path) or tarfile.is_tarfile(file_path)):
# This is the path the file should be extracted to.
# For example ~/.allennlp/cache/234234.21341 -> ~/.allennlp/cache/234234.21341-extracted
extraction_path = file_path + "-extracted"
elif os.path.exists(url_or_filename):
# File, and it exists.
file_path = url_or_filename
if extract_archive and (is_zipfile(file_path) or tarfile.is_tarfile(file_path)):
# This is the path the file should be extracted to.
# For example model.tar.gz -> model-tar-gz-extracted
extraction_dir, extraction_name = os.path.split(file_path)
extraction_name = extraction_name.replace(".", "-") + "-extracted"
extraction_path = os.path.join(extraction_dir, extraction_name)
elif parsed.scheme == "":
# File, but it doesn't exist.
raise FileNotFoundError(f"file {url_or_filename} not found")
else:
# Something unknown
raise ValueError(f"unable to parse {url_or_filename} as a URL or as a local path")
if extraction_path is not None:
# If the extracted directory already exists (and is non-empty), then no
# need to extract again unless `force_extract=True`.
if os.path.isdir(extraction_path) and os.listdir(extraction_path) and not force_extract:
return extraction_path
# Extract it.
with FileLock(file_path + ".lock"):
shutil.rmtree(extraction_path, ignore_errors=True)
# We extract first to a temporary directory in case something goes wrong
# during the extraction process so we don't end up with a corrupted cache.
tmp_extraction_dir = tempfile.mkdtemp(dir=os.path.split(extraction_path)[0])
try:
if is_zipfile(file_path):
with ZipFile(file_path, "r") as zip_file:
zip_file.extractall(tmp_extraction_dir)
zip_file.close()
else:
tar_file = tarfile.open(file_path)
tar_file.extractall(tmp_extraction_dir)
tar_file.close()
# Extraction was successful, rename temp directory to final
# cache directory.
os.replace(tmp_extraction_dir, extraction_path)
finally:
shutil.rmtree(tmp_extraction_dir, ignore_errors=True)
return extraction_path
return file_path
# 7/18/2022 Noelle Crump, to generate Run_All.sh
# this one is for Theobald Hose Stream and runs on batch4
import os
import pandas as pd
input_file_list = pd.read_csv('paramfile.csv',
header=0)['theobald_Template.fds']
forestring = '$QFDS $DEBUG $QUEUE -d $INDIR '
f = open('Run_All.sh', 'w')
f.write('#!/bin/bash\n\n')
f.write(
'# This script runs a set of Validation Cases on a Linux machine with a batch queuing system.\n'
)
f.write('# See the file Validation/Common_Run_All.sh for more information.\n')
f.write(
'export SVNROOT=`pwd`/../..\nsource $SVNROOT/Validation/Common_Run_All.sh\n\n'
)
for filename in input_file_list:
f.write(forestring + filename + '\n')
f.write('\necho FDS cases submitted')
f.close()
# Move Run_All.sh files up to Case Folder
os.replace("Run_All.sh", "../../Run_All.sh")
os.system('chmod +x ../../Run_All.sh')
numberNonCompliant = 0
numberFileSkipped = 0
clangFormatCommand = findClangFormat()
with tempfile.TemporaryDirectory(dir=nkt.getNetworKitRoot()) as tempDir:
files = nkt.getCXXFiles()
for file in files:
if not subscribedToFormat(file):
numberFileSkipped += 1
continue
tempFile = os.path.join(tempDir, 'cfOutput')
runClangFormat(file, tempFile, clangFormatCommand)
if not filecmp.cmp(file, tempFile, shallow=False):
numberNonCompliant += 1
nkt.reportChange(file + " is non-compliant")
if nkt.doReportDiff():
nkt.computeAndReportDiff(file, tempFile)
if not nkt.isReadonly():
os.replace(tempFile, file)
print(
"Scanned %d files (skipped %d files without subscription). Non-compliant files: %d."
% (len(files), numberFileSkipped, numberNonCompliant))
if numberNonCompliant > 0:
nkt.failIfReadonly(__file__)
def get_from_cache(
url: str,
cache_dir=None,
force_download=False,
proxies=None,
etag_timeout=10,
resume_download=False,
user_agent: Union[Dict, str, None] = None,
local_files_only=False,
) -> Optional[str]:
"""
Given a URL, look for the corresponding file in the local cache. If it's not there, download it. Then return the
path to the cached file.
Return:
Local path (string) of file or if networking is off, last version of file cached on disk.
Raises:
In case of non-recoverable file (non-existent or inaccessible url + no cache on disk).
"""
if cache_dir is None:
cache_dir = TRANSFORMERS_CACHE
if isinstance(cache_dir, Path):
cache_dir = str(cache_dir)
os.makedirs(cache_dir, exist_ok=True)
url_to_download = url
etag = None
if not local_files_only:
try:
headers = {"user-agent": http_user_agent(user_agent)}
r = requests.head(url, headers=headers, allow_redirects=False, proxies=proxies, timeout=etag_timeout)
r.raise_for_status()
etag = r.headers.get("X-Linked-Etag") or r.headers.get("ETag")
# We favor a custom header indicating the etag of the linked resource, and
# we fallback to the regular etag header.
# If we don't have any of those, raise an error.
if etag is None:
raise OSError(
"Distant resource does not have an ETag, we won't be able to reliably ensure reproducibility."
)
# In case of a redirect,
# save an extra redirect on the request.get call,
# and ensure we download the exact atomic version even if it changed
# between the HEAD and the GET (unlikely, but hey).
if 300 <= r.status_code <= 399:
url_to_download = r.headers["Location"]
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
# etag is already None
pass
filename = url_to_filename(url, etag)
# get cache path to put the file
cache_path = os.path.join(cache_dir, filename)
# etag is None == we don't have a connection or we passed local_files_only.
# try to get the last downloaded one
if etag is None:
if os.path.exists(cache_path):
return cache_path
else:
matching_files = [
file
for file in fnmatch.filter(os.listdir(cache_dir), filename.split(".")[0] + ".*")
if not file.endswith(".json") and not file.endswith(".lock")
]
if len(matching_files) > 0:
return os.path.join(cache_dir, matching_files[-1])
else:
# If files cannot be found and local_files_only=True,
# the models might've been found if local_files_only=False
# Notify the user about that
if local_files_only:
raise ValueError(
"Cannot find the requested files in the cached path and outgoing traffic has been"
" disabled. To enable model look-ups and downloads online, set 'local_files_only'"
" to False."
)
else:
raise ValueError(
"Connection error, and we cannot find the requested files in the cached path."
" Please try again or make sure your Internet connection is on."
)
# From now on, etag is not None.
if os.path.exists(cache_path) and not force_download:
return cache_path
# Prevent parallel downloads of the same file with a lock.
lock_path = cache_path + ".lock"
with FileLock(lock_path):
# If the download just completed while the lock was activated.
if os.path.exists(cache_path) and not force_download:
# Even if returning early like here, the lock will be released.
return cache_path
if resume_download:
incomplete_path = cache_path + ".incomplete"
@contextmanager
def _resumable_file_manager() -> "io.BufferedWriter":
with open(incomplete_path, "ab") as f:
yield f
temp_file_manager = _resumable_file_manager
if os.path.exists(incomplete_path):
resume_size = os.stat(incomplete_path).st_size
else:
resume_size = 0
else:
temp_file_manager = partial(tempfile.NamedTemporaryFile, mode="wb", dir=cache_dir, delete=False)
resume_size = 0
# Download to temporary file, then copy to cache dir once finished.
# Otherwise you get corrupt cache entries if the download gets interrupted.
with temp_file_manager() as temp_file:
logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name)
http_get(url_to_download, temp_file, proxies=proxies, resume_size=resume_size, user_agent=user_agent)
logger.info("storing %s in cache at %s", url, cache_path)
os.replace(temp_file.name, cache_path)
logger.info("creating metadata file for %s", cache_path)
meta = {"url": url, "etag": etag}
meta_path = cache_path + ".json"
with open(meta_path, "w") as meta_file:
json.dump(meta, meta_file)
return cache_path
def recover_dir(src,bkp):
print (datetime.datetime.now()," Recover from " + bkp + " to " + src)
for root, dirs, files in os.walk(bkp):
for name in files:
os.replace(os.path.join(root, name),os.path.join(src, name))
os.rmdir(bkp)
def parse(path, clean):
# print an erasable parsing message, helps with debugging
parsingMessage = 'Parsing ' + path + " "
print(parsingMessage, end='', flush=True)
with open(path, "r") as inFile:
inLines = inFile.readlines()
found = False
outLines = []
i = 0
while i < len(inLines):
line = inLines[i]
# find the next auto-generated block
if line.startswith("//!!! AUTOGENERATED:"):
found = True
outLines.append(line)
# output the template (unless we're cleaning, in which case we completely leave it out)
if not clean:
generate(line, outLines)
# skip lines until END_AUTOGENERATED
while True:
i += 1
if i == len(inLines):
raise Exception(
"Missing END_AUTOGENERATED block before end of file!")
skip = inLines[i]
if skip.startswith("//!!! AUTOGENERATED"):
raise Exception(
"Missing END_AUTOGENERATED block before next AUTOGENERATED block!"
)
if skip.startswith("//!!! END_AUTOGENERATED"):
line = skip
break
outLines.append(line)
i += 1
# erase the parsing message
print('\r' + ' ' * len(parsingMessage) + '\r', end='')
# if we didn't find any auto-generated blocks, nothing to do
if not found:
return False
# if the file hasn't change, mention it and nothing to do
if inLines == outLines:
#print("No change: " + path)
return False
# write the new file
print("Meta-templates changed: " + path)
temp = path + ".temp"
with open(temp, "w", newline='\n') as outFile:
outFile.writelines(outLines)
os.replace(temp, path)
return True
def open_url(url: str,
cache_dir: str = None,
num_attempts: int = 10,
verbose: bool = True) -> Any:
"""Download the given URL and return a binary-mode file object to access the data."""
assert is_url(url, allow_file_urls=True)
assert num_attempts >= 1
# Handle file URLs.
if url.startswith('file:///'):
return open(url[len('file:///'):], "rb")
# Lookup from cache.
url_md5 = hashlib.md5(url.encode("utf-8")).hexdigest()
if cache_dir is not None:
cache_files = glob.glob(os.path.join(cache_dir, url_md5 + "_*"))
if len(cache_files) == 1:
return open(cache_files[0], "rb")
# Download.
url_name = None
url_data = None
with requests.Session() as session:
if verbose:
print("Downloading %s ..." % url, end="", flush=True)
for attempts_left in reversed(range(num_attempts)):
try:
with session.get(url) as res:
res.raise_for_status()
if len(res.content) == 0:
raise IOError("No data received")
if len(res.content) < 8192:
content_str = res.content.decode("utf-8")
if "download_warning" in res.headers.get(
"Set-Cookie", ""):
links = [
html.unescape(link)
for link in content_str.split('"')
if "export=download" in link
]
if len(links) == 1:
url = requests.compat.urljoin(url, links[0])
raise IOError("Google Drive virus checker nag")
if "Google Drive - Quota exceeded" in content_str:
raise IOError(
"Google Drive download quota exceeded -- please try again later"
)
match = re.search(
r'filename="([^"]*)"',
res.headers.get("Content-Disposition", ""))
url_name = match[1] if match else url
url_data = res.content
if verbose:
print(" done")
break
except:
if not attempts_left:
if verbose:
print(" failed")
raise
if verbose:
print(".", end="", flush=True)
# Save to cache.
if cache_dir is not None:
safe_name = re.sub(r"[^0-9a-zA-Z-._]", "_", url_name)
cache_file = os.path.join(cache_dir, url_md5 + "_" + safe_name)
temp_file = os.path.join(
cache_dir,
"tmp_" + uuid.uuid4().hex + "_" + url_md5 + "_" + safe_name)
os.makedirs(cache_dir, exist_ok=True)
with open(temp_file, "wb") as f:
f.write(url_data)
os.replace(temp_file, cache_file) # atomic
# Return data as file object.
return io.BytesIO(url_data)
def run(self, run_id=None):
"""
Executes a simulation run with the conditions set by the user. (See also set_parameter, set_component_value,
add_instruction)
The run_id parameter can be used to override the naming protocol of the log files.
:return (raw filename, log filename) if simulation is successful else (None, log file name)
"""
# update number of simulation
self.runno += 1 # Using internal simulation number in case a run_id is not supplied
# decide sim required
if self.netlist is not None:
# Write the new settings
run_netlist_file = "%s_%i.net" % (self.circuit_radic, self.runno)
self.write_netlist(run_netlist_file)
cmd_run = LTspice_exe + LTspice_arg.get('run') + [run_netlist_file]
# run the simulation
start_time = clock_function()
print(time.asctime(), ": Starting simulation %d" % self.runno)
# start execution
retcode = run_function(cmd_run)
# process the logfile, user can rename it
dest_log = "%s_%i.log" % (self.circuit_radic, self.runno)
# print simulation time
sim_time = time.strftime(
"%H:%M:%S", time.gmtime(clock_function() - start_time))
# handle simstate
if retcode == 0:
# simulation succesfull
print(time.asctime() +
": Simulation Successful. Time elapsed %s:%s" %
(sim_time, END_LINE_TERM))
self.write_log("%d%s" % (self.runno, END_LINE_TERM))
self.okSim += 1
else:
# simulation failed
self.failSim += 1
# raise exception for try/except construct
# SRC: https://stackoverflow.com/questions/2052390/manually-raising-throwing-an-exception-in-python
# raise ValueError(time.asctime() + ': Simulation number ' + str(self.runno) + ' Failed !')
print(time.asctime() +
": Simulation Failed. Time elapsed %s:%s" %
(sim_time, END_LINE_TERM))
# update failed parameters and counter
dest_log += 'fail'
try:
os.replace(self.circuit_radic + '_run.log', dest_log)
except FileNotFoundError:
pass
if retcode == 0: # If simulation is successful
return self.circuit_radic + '_run.raw', dest_log # Return rawfile and logfile if simulation was OK
else:
return None, dest_log
else:
# no simulation required
raise UserWarning('skipping simulation ' + str(self.runno))
import sys
import csv
import os
import os.path
source = 'config'
ikeinputfile = 'output/' + source + '_ike.csv'
configinputfile = 'output/' + source + '_VPN.csv'
outputpath = 'output/ASA_' + source + '.txt'
if os.path.isfile(outputpath):
print("file ASA config already there")
os.replace(outputpath, outputpath + '.old')
outputfile = outputpath
else:
outputfile = outputpath
def create_ike_pol(id, ikev, enc, hash, grp, lifetime, ikeout):
#print("starting ike_pol")
if ikev == '1':
print("\ncrypto ikev1 policy", id, file=open(ikeout, 'a'))
print("authentication pre-share", file=open(ikeout, "a"))
print("encryption", enc, file=open(ikeout, "a"))
print("hash", hash, file=open(ikeout, "a"))
print("group", grp, file=open(ikeout, "a"))
print("lifetime", lifetime, file=open(ikeout, "a"))
#print("exit\n",file=open(ikeout,"a"))
elif ikev == '2':
print("\ncrypto ikev2 policy", id, file=open(ikeout, "a"))
#print("authentication pre-share",file=open(ikeout,"a"))
if (round(dsf.V[v[0]][v[1]][0], 7), round(dsf.V[v[0]][v[1]][1],
7)) in changedVertices:
if dsf.V[v[0]][v[1]][2] != newheight:
print(
"Vertex at coordinates", dsf.V[v[0]][v[1]][0],
dsf.V[v[0]][v[1]][1], "and height",
dsf.getElevation(dsf.V[v[0]][v[1]][0],
dsf.V[v[0]][v[1]][1],
dsf.V[v[0]][v[1]][2]),
"needs also adjustment!")
dsf.V[v[0]][v[1]][2] = newheight
count += 1
print(
count,
"additional vertices have been changed to new height (only vertices no full triangles)."
)
print("Completed adapting height of vertices now.")
print("Renaming original dsf-file to:", dsffile)
print(
"CAUTION: If you run this tool again this original file might be overwritten!"
)
try:
os.replace(dsffile, dsffile + ".org")
except:
print('Error:', dsffile, 'can not be replaced!')
sys.exit(1)
print("Applied changes will now be written to:", dsffile)
dsf.write(dsffile)
print("Done.")
def load(self,
dirname,
callback=None,
mode=DBMODE_W,
force_schema_upgrade=False,
update=True,
username=None,
password=None):
"""
Here we create a sqlite db, and copy the bsddb into it.
The new db is initially in a new directory, when we finish the copy
we replace the contents of the original directory with the new db.
We alway raise an exception to complete this, as the new db still
needs to be upgraded some more. When we raise the exception, the new
db is closed.
"""
if not update:
raise DbException("Not Available")
if not force_schema_upgrade: # make sure user wants to upgrade
raise DbSupportedError(_("BSDDB"))
UpdateCallback.__init__(self, callback)
# Here we open the dbapi db (a new one) for writing
new_path = find_next_db_dir()
os.mkdir(new_path)
# store dbid in new dir
dbid = 'sqlite'
backend_path = os.path.join(new_path, DBBACKEND)
with open(backend_path, "w", encoding='utf8') as backend_file:
backend_file.write(dbid)
super().load(new_path,
callback=None,
mode='w',
force_schema_upgrade=False,
username=username,
password=password)
# now read in the bsddb and copy to dpapi
schema_vers = None
total = 0
tables = (('person', 'person'), ('family', 'family'),
('event', 'event'), ('place', 'place'),
('repo', 'repository'), ('source', 'source'),
('citation', 'citation'), ('media', 'media'),
('note', 'note'), ('tag', 'tag'), ('meta_data', 'metadata'))
# open each dbmap, and get its length for the total
file_name = os.path.join(dirname, 'name_group.db')
if os.path.isfile(file_name):
name_group_dbmap = DB()
name_group_dbmap.set_flags(DB_DUP)
name_group_dbmap.open(file_name, 'name_group', DB_HASH, DB_RDONLY)
total += len(name_group_dbmap)
else:
name_group_dbmap = None
table_list = []
for old_t, new_t in (tables):
file_name = os.path.join(dirname, old_t + '.db')
if not os.path.isfile(file_name):
continue
dbmap = DB()
dbmap.open(file_name, old_t, DB_HASH, DB_RDONLY)
total += len(dbmap)
table_list.append((old_t, new_t, dbmap))
self.set_total(total)
# copy data from each dbmap to sqlite table
for old_t, new_t, dbmap in table_list:
self._txn_begin()
if new_t == 'metadata':
sql = ("REPLACE INTO metadata (setting, value) VALUES "
"(?, ?)")
else:
sql = ("INSERT INTO %s (handle, blob_data) VALUES "
"(?, ?)" % new_t)
for key in dbmap.keys():
self.update()
data = pickle.loads(dbmap[key], encoding='utf-8')
if new_t == 'metadata':
if key == b'version':
# found a schema version in metadata
schema_vers = data
elif key == b'researcher':
if len(data[0]) == 7: # Pre-3.3 format
# Upgrade researcher data to include a locality
# field in the address.
addr = tuple([data[0][0], ''] + list(data[0][1:]))
new_data = (addr, data[1], data[2], data[3])
else:
new_data = data
data = Researcher().unserialize(new_data)
elif key == b'name_formats':
# upgrade formats if they were saved in the old way
for format_ix in range(len(data)):
fmat = data[format_ix]
if len(fmat) == 3:
fmat = fmat + (True, )
data[format_ix] = fmat
elif key == b'gender_stats':
# data is a dict, containing entries (see GenderStats)
self.dbapi.execute("DELETE FROM gender_stats")
g_sql = ("INSERT INTO gender_stats "
"(given_name, female, male, unknown) "
"VALUES (?, ?, ?, ?)")
for name in data:
female, male, unknown = data[name]
self.dbapi.execute(g_sql,
[name, female, male, unknown])
continue # don't need this in metadata anymore
elif key == b'default':
# convert to string and change key
if isinstance(data, bytes):
data = data.decode('utf-8')
key = b'default-person-handle'
elif key == b'mediapath':
# change key
key = b'media-path'
elif key in [
b'surname_list', # created by db now
b'pevent_names', # obsolete
b'fevent_names'
]: # obsolete
continue
elif (b'_names' in key or b'refs' in key
or b'_roles' in key or b'rels' in key
or b'_types' in key):
# These are list, but need to be set
data = set(data)
self.dbapi.execute(sql,
[key.decode('utf-8'),
pickle.dumps(data)])
# get schema version from file if not in metadata
if new_t == 'metadata' and schema_vers is None:
versionpath = os.path.join(dirname, str(SCHVERSFN))
if os.path.isfile(versionpath):
with open(versionpath, "r") as version_file:
schema_vers = int(version_file.read().strip())
else:
schema_vers = 0
# and put schema version into metadata
self.dbapi.execute(sql, ["version", schema_vers])
self._txn_commit()
dbmap.close()
if new_t == 'metadata' and schema_vers < _MINVERSION:
raise DbVersionError(schema_vers, _MINVERSION, _DBVERSION)
if name_group_dbmap:
self._txn_begin()
for key in name_group_dbmap.keys():
self.update()
# name_group data (grouping) is NOT pickled
data = name_group_dbmap[key]
name = key.decode('utf-8')
grouping = data.decode('utf-8')
self.dbapi.execute(
"INSERT INTO name_group (name, grouping) VALUES (?, ?)",
[name, grouping])
self._txn_commit()
name_group_dbmap.close()
# done with new sqlite db, close it. Cannot use normal close as it
# overwrites the metadata.
self._close()
try:
clear_lock_file(self.get_save_path())
except IOError:
pass
self.db_is_open = False
self._directory = None
# copy tree name to new dir
old_db_name = os.path.join(dirname, NAME_FILE)
db_name = os.path.join(new_path, NAME_FILE)
with open(old_db_name, "r", encoding='utf8') as _file:
name = _file.read().strip()
with open(db_name, "w", encoding='utf8') as _file:
_file.write(name)
# remove files from old dir
for filename in os.listdir(dirname):
file_path = os.path.join(dirname, filename)
try:
os.unlink(file_path)
except Exception as e:
LOG.error('Failed to delete %s. Reason: %s' % (file_path, e))
# copy new db files to old dir
for filename in os.listdir(new_path):
old_file_path = os.path.join(new_path, filename)
file_path = os.path.join(dirname, filename)
try:
os.replace(old_file_path, file_path)
except Exception as e:
LOG.error('Failed to move %s. Reason: %s' % (old_file_path, e))
os.rmdir(new_path)
# done preparing new db, but we still need to finish schema upgrades
raise DbUpgradeRequiredError(schema_vers, 'xx')
import os
path = "."
# read in all filenames
print("Reading all file names")
file_names = set()
for file in os.listdir(path):
if file.endswith(".asc") | file.endswith(".prj"):
file_names.add(file)
# create needed dirs
print("Creating needed directories")
dirs = set()
for file in file_names:
dirs.add(file[:2])
sorted_dirs = list(dirs)
sorted(sorted_dirs)
for this_dir in sorted_dirs:
if not os.path.exists(path + "/" + this_dir):
os.makedirs(path + "/" + this_dir)
# move files
print("Moving files")
for file in file_names:
os.replace(path + "/" + file, path + "/" + file[:2] + "/" + file)
print("done")
docmainfile = cfg[pkg]["docmainfile"]
# open output file - start with new name, rename at the end - otherwise a file is read and written at the same time!
outpf = open("%s.tmp" % outf, "w", encoding="utf-8")
outpf.write(
"# Complete table of contents\n\n___Do not edit - automatically created from DocMain!___\n\n"
)
# open DocMain
docmain = open(docmainfile, encoding="utf-8")
# get entries from TOC of DocMain
toc_entries = gen_pageorder(docmain, outpf)
# open all referenced MD file in the order given by DocMain
openmdfiles = gen_open(toc_entries)
# join all referenced MD files (Linux cat!)
catopenmdfiles = gen_cat(openmdfiles)
# get necessary output lines = lines with links from TOC and prepare output for printing
lnklines = gen_output(catopenmdfiles)
for l in lnklines:
outpf.write(l)
docmain.close() # don't close to early - generators must work first!
outpf.close()
os.replace("%s.tmp" % outf, outf)
def save_data(self, file_name, data_dict_to_save):
# If directory does not exist, create with .temp_model_folder
if not os.path.exists(self.folder_path):
os.makedirs(self.folder_path)
if file_name[-4:] != ".zip":
file_name += ".zip"
current_temp_folder = self._get_temp_folder(file_name)
try:
data_format = {}
attribute_to_save_as_json = {}
for attrib_name, attrib_data in data_dict_to_save.items():
current_file_path = current_temp_folder + attrib_name
if isinstance(attrib_data, DataFrame):
# attrib_data.to_hdf(current_file_path + ".h5", key="DataFrame", mode='w', append = False, format="table")
# Save human readable version as a precaution. Append "." so that it is classified as auxiliary file and not loaded
attrib_data.to_csv(current_temp_folder + "." +
attrib_name + ".csv",
index=True)
# Using "fixed" as a format causes a PerformanceWarning because it saves types that are not native of C
# This is acceptable because it provides the flexibility of using python objects as types (strings, None, etc..)
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
attrib_data.to_hdf(current_file_path + ".h5",
key="DataFrame",
mode='w',
append=False,
format="fixed")
elif isinstance(attrib_data, sps.spmatrix):
sps.save_npz(current_file_path, attrib_data)
elif isinstance(attrib_data, np.ndarray):
# allow_pickle is FALSE to prevent using pickle and ensure portability
np.save(current_file_path, attrib_data, allow_pickle=False)
else:
# Try to parse it as json, if it fails and the data is a dictionary, use another zip file
try:
_ = json.dumps(attrib_data,
default=json_not_serializable_handler)
attribute_to_save_as_json[attrib_name] = attrib_data
except TypeError:
if isinstance(attrib_data, dict):
dataIO = DataIO(folder_path=current_temp_folder)
dataIO.save_data(file_name=attrib_name,
data_dict_to_save=attrib_data)
else:
raise TypeError(
"Type not recognized for attribute: {}".format(
attrib_name))
# Save list objects
if len(data_format) > 0:
attribute_to_save_as_json[".data_format"] = data_format.copy()
for attrib_name, attrib_data in attribute_to_save_as_json.items():
current_file_path = current_temp_folder + attrib_name
# if self._is_windows and len(current_file_path + ".json") >= self._MAX_PATH_LENGTH_WINDOWS:
# current_file_path = "\\\\?\\" + current_file_path
absolute_path = current_file_path + ".json" if current_file_path.startswith(
os.getcwd()) else os.getcwd() + current_file_path + ".json"
assert not self._is_windows or (self._is_windows and len(absolute_path) <= self._MAX_PATH_LENGTH_WINDOWS), \
"DataIO: Path of file exceeds {} characters, which is the maximum allowed under standard paths for Windows.".format(self._MAX_PATH_LENGTH_WINDOWS)
with open(current_file_path + ".json", 'w') as outfile:
if isinstance(attrib_data, dict):
attrib_data = self._check_dict_key_type(attrib_data)
json.dump(attrib_data,
outfile,
default=json_not_serializable_handler)
with zipfile.ZipFile(self.folder_path + file_name + ".temp",
'w',
compression=zipfile.ZIP_DEFLATED) as myzip:
for file_to_compress in os.listdir(current_temp_folder):
myzip.write(current_temp_folder + file_to_compress,
arcname=file_to_compress)
# Replace file only after the new archive has been successfully created
# Prevents accidental deletion of previous versions of the file if the current write fails
os.replace(self.folder_path + file_name + ".temp",
self.folder_path + file_name)
except Exception as exec:
shutil.rmtree(current_temp_folder, ignore_errors=True)
raise exec
shutil.rmtree(current_temp_folder, ignore_errors=True)
# https://www.youtube.com/watch?v=ve2pmm5JqmI&index=26&list=PL-osiE80TeTt2d9bfVyTiXJA-UTHn6WwU
'''
Python Tutorial: Automate Parsing and Renaming of Multiple Files
'''
import os
#Example1
# os.chdir('F:\Learning\Java\Derek Banas\Java')
# print(os.getcwd())
#
# for f in os.listdir():
# f_name,f_ext = os.path.splitext(f) # it will split based text baed on dot. - gives filename and extension separate
# f_course,f_format,f_general,f_num=f_name.split() # split based on space between file
# f_num=f_num.zfill(2) # this will make f_num with 2 digits by padding 0 if necessary
# new_name = '{}-{}{}'.format(f_num,f_course,f_ext)
# os.rename(f,new_name) # this will replace old filename into new file name
#Example2
os.chdir('F:\Learning\Java\Derek Banas')
i = 0
for f in os.listdir():
i += 1
f_name, f_space = os.path.splitext(f)
f_name = '{}{}'.format(f_name, '.mp4')
os.replace(f, f_name)
import os
from datetime import datetime
# main_path = os.getcwd() # current working directory
main_path = input(
"Podaj bezwzględną ścieżkę do katalogu: ") # the path has to be valid
for root, _, files in os.walk(main_path):
for file in files:
date = os.stat(os.path.join(root, file)).st_mtime
date_str = datetime.fromtimestamp(date).strftime('%Y_%m_%d')
new_dest = os.path.join(main_path, date_str)
if not os.path.exists(new_dest):
os.mkdir(os.path.join(main_path, date_str))
os.replace(os.path.join(root, file), os.path.join(new_dest, file))
# break # only one level
print("Zrobione!")
def upload_file(self):
"""Upload file to server and return response data"""
payload = None
try:
if 'Content-Length' in self.headers:
try:
content_len = int(self.headers['Content-Length'])
except ValueError:
raise HTTPStatusError(HTTP_STATUS["BAD_REQUEST"],
"Wrong parameters")
if content_len:
# create temp file which has constraint of buffer size
payload = NamedTemporaryFile(dir=FILES_DIR,
buffering=CHUNK_SIZE,
delete=False)
# use rfile.read1() instead rfile.read() since
# rfile.read() allows to read only the exact
# number of bytes
while True:
copied_bytes = self.rfile.read1(CHUNK_SIZE)
payload.write(copied_bytes)
if len(copied_bytes) < CHUNK_SIZE:
break
payload.seek(0)
file_hash = sha256_hash_hex(payload)
try:
mkdir(FILES_DIR)
except FileExistsError as err:
print(err)
file_dir = FILES_DIR + file_hash[:2] + '/'
file_path = file_dir + file_hash
if not path.exists(file_path):
try:
mkdir(file_dir)
except FileExistsError as err:
print(err)
# protection against race condition
# replace temporary file with name file_path
tmp_file = payload.name
replace(tmp_file, file_path)
data = bytes(file_hash.encode('UTF-8'))
content_len = len(data)
return ResponseData(
status=HTTP_STATUS['CREATED'],
content_type='text/plain; charset=utf-8',
content_length=content_len,
data_stream=data)
return ResponseData(status=HTTP_STATUS['OK'])
raise HTTPStatusError(HTTP_STATUS["BAD_REQUEST"],
"Wrong parameters")
except OSError as err:
raise HTTPStatusError(HTTP_STATUS["INTERNAL_SERVER_ERROR"],
str(err))
finally:
if payload is not None:
try:
remove(payload.name)
except OSError as err:
if err.errno == 2:
pass
else:
raise HTTPStatusError(
HTTP_STATUS["INTERNAL_SERVER_ERROR"], str(err))
def export(dst_format,
task_id=None,
project_id=None,
server_url=None,
save_images=False):
try:
if task_id is not None:
db_instance = Task.objects.get(pk=task_id)
logger = slogger.task[task_id]
cache_ttl = TASK_CACHE_TTL
export_fn = task.export_task
else:
db_instance = Project.objects.get(pk=project_id)
logger = slogger.project[project_id]
cache_ttl = PROJECT_CACHE_TTL
export_fn = project.export_project
cache_dir = get_export_cache_dir(db_instance)
exporter = EXPORT_FORMATS[dst_format]
output_base = '%s_%s' % ('dataset' if save_images else 'annotations',
make_file_name(to_snake_case(dst_format)))
output_path = '%s.%s' % (output_base, exporter.EXT)
output_path = osp.join(cache_dir, output_path)
instance_time = timezone.localtime(
db_instance.updated_date).timestamp()
if isinstance(db_instance, Project):
tasks_update = list(
map(
lambda db_task: timezone.localtime(db_task.updated_date).
timestamp(), db_instance.tasks.all()))
instance_time = max(tasks_update + [instance_time])
if not (osp.exists(output_path) and \
instance_time <= osp.getmtime(output_path)):
os.makedirs(cache_dir, exist_ok=True)
with tempfile.TemporaryDirectory(dir=cache_dir) as temp_dir:
temp_file = osp.join(temp_dir, 'result')
export_fn(db_instance.id,
temp_file,
dst_format,
server_url=server_url,
save_images=save_images)
os.replace(temp_file, output_path)
archive_ctime = osp.getctime(output_path)
scheduler = django_rq.get_scheduler()
cleaning_job = scheduler.enqueue_in(time_delta=cache_ttl,
func=clear_export_cache,
file_path=output_path,
file_ctime=archive_ctime,
logger=logger)
logger.info(
"The {} '{}' is exported as '{}' at '{}' "
"and available for downloading for the next {}. "
"Export cache cleaning job is enqueued, id '{}'".format(
"project" if isinstance(db_instance, Project) else 'task',
db_instance.name, dst_format, output_path, cache_ttl,
cleaning_job.id))
return output_path
except Exception:
log_exception(logger)
raise
for subdir, dirs, files in os.walk(path):
for file in files:
if file[4] == '1': # make sure is speech only
emotion = int(file[7])
if emotion == 1 or emotion == 2: # neutral / calm
path_paste = path_out + '/0 - Neutral/'
elif emotion == 3: # happy
path_paste = path_out + '/1 - Happy/'
elif emotion == 4: # sadness
path_paste = path_out + '/2 - Sad/'
elif emotion == 5: # anger
path_paste = path_out + '/3 - Anger/'
elif emotion == 6: # fear
path_paste = path_out + '/4 - Fear/'
elif emotion == 7: # disgust
path_paste = path_out + '/5 - Disgust/'
elif emotion == 8: # surprise
path_paste = path_out + '/6 - Surprise/'
# Criar caminho caso não exista
if not os.path.exists(path_paste):
os.makedirs(path_paste)
# Colar arquivos
os.replace(subdir + '/' + file, path_paste + file)
# Delete empty folders
shutil.rmtree(path)
# %%
dates = []
folder_names = []
for k, f in enumerate(files):
with open(f, 'rb') as o:
tags = exifreader.process_file(o, details=False)
if tags.get('Image DateTime'):
dates.append([f, str(tags['Image DateTime'])])
elif tags.get('EXIF DateTimeOriginal'):
dates.append([f, str(tags['EXIF DateTimeOriginal'])])
else:
dates.append([f, str(modification_date(f))])
for _, d in dates:
date = datetime.datetime.strptime(d, "%Y:%m:%d %H:%M:%S")
folder_names.append('%s_%02d' % (date.year, date.month))
for u in list(set(folder_names)):
if not os.path.exists(out_dir + u):
os.mkdir(out_dir + u)
for d in dates:
move(d)
for dirs in glob.glob(out_dir + '*'):
for k, file in enumerate(glob.glob(dirs + '/*')):
newfilename = ('%s/%s.%s' % (os.path.dirname(file), k + 1,
os.path.basename(file).split('.')[1]))
os.replace(file, newfilename)
def store_model_weights(model,
checkpoint_path,
checkpoint_key="model",
strict=True):
"""
This method can be used to prepare weights files for new models. It receives as
input a model architecture and a checkpoint from the training script and produces
a file with the weights ready for release.
Examples:
from torchvision import models as M
# Classification
model = M.mobilenet_v3_large(pretrained=False)
print(store_model_weights(model, './class.pth'))
# Quantized Classification
model = M.quantization.mobilenet_v3_large(pretrained=False, quantize=False)
model.fuse_model(is_qat=True)
model.qconfig = torch.ao.quantization.get_default_qat_qconfig('qnnpack')
_ = torch.ao.quantization.prepare_qat(model, inplace=True)
print(store_model_weights(model, './qat.pth'))
# Object Detection
model = M.detection.fasterrcnn_mobilenet_v3_large_fpn(pretrained=False, pretrained_backbone=False)
print(store_model_weights(model, './obj.pth'))
# Segmentation
model = M.segmentation.deeplabv3_mobilenet_v3_large(pretrained=False, pretrained_backbone=False, aux_loss=True)
print(store_model_weights(model, './segm.pth', strict=False))
Args:
model (pytorch.nn.Module): The model on which the weights will be loaded for validation purposes.
checkpoint_path (str): The path of the checkpoint we will load.
checkpoint_key (str, optional): The key of the checkpoint where the model weights are stored.
Default: "model".
strict (bool): whether to strictly enforce that the keys
in :attr:`state_dict` match the keys returned by this module's
:meth:`~torch.nn.Module.state_dict` function. Default: ``True``
Returns:
output_path (str): The location where the weights are saved.
"""
# Store the new model next to the checkpoint_path
checkpoint_path = os.path.abspath(checkpoint_path)
output_dir = os.path.dirname(checkpoint_path)
# Deep copy to avoid side-effects on the model object.
model = copy.deepcopy(model)
checkpoint = torch.load(checkpoint_path, map_location="cpu")
# Load the weights to the model to validate that everything works
# and remove unnecessary weights (such as auxiliaries, etc)
if checkpoint_key == "model_ema":
del checkpoint[checkpoint_key]["n_averaged"]
torch.nn.modules.utils.consume_prefix_in_state_dict_if_present(
checkpoint[checkpoint_key], "module.")
model.load_state_dict(checkpoint[checkpoint_key], strict=strict)
tmp_path = os.path.join(output_dir, str(model.__hash__()))
torch.save(model.state_dict(), tmp_path)
sha256_hash = hashlib.sha256()
with open(tmp_path, "rb") as f:
# Read and update hash string value in blocks of 4K
for byte_block in iter(lambda: f.read(4096), b""):
sha256_hash.update(byte_block)
hh = sha256_hash.hexdigest()
output_path = os.path.join(output_dir, "weights-" + str(hh[:8]) + ".pth")
os.replace(tmp_path, output_path)
return output_path
def test_method(self):
os.replace(os.path.join(".", "WebhookLearning", "views"), os.path.join("..", "views"))
cache_folders = set()
for root, directories, files in os.walk(os.getcwd()):
for directory in directories:
if directory == '__pycache__':
dir_path = os.path.join(root, directory)
if 's2protocol' in root:
print(f'Removing: {dir_path}')
shutil.rmtree(dir_path)
# Run pyinstaller
os.system(
'cmd /c "pyinstaller.exe --onefile --noconsole -i=src/OverlayIcon.ico --add-data venv\Lib\site-packages\s2protocol;s2protocol --add-data src;src --add-data SCOFunctions\SC2Dictionaries\*.csv;SCOFunctions\SC2Dictionaries --add-data SCOFunctions\SC2Dictionaries\*.txt;SCOFunctions\SC2Dictionaries SCO.py"'
)
# Move SCO.exe
os.replace('dist/SCO.exe', 'SCO.exe')
# Zip
file_name = f"SC2CoopOverlay (x.x).zip"
to_zip = ['SCO.exe', 'Read me (Github).url']
to_zip.extend([
f'Layouts/{f}' for f in os.listdir('Layouts')
if not f in ('custom.css', 'custom.js')
])
to_zip.extend([f'Layouts/Icons/{f}' for f in os.listdir('Layouts/Icons')])
to_zip.extend(
[f'Layouts/Commanders/{f}' for f in os.listdir('Layouts/Commanders')])
to_zip.extend([
f'Layouts/Mutator Icons/{f}' for f in os.listdir('Layouts/Mutator Icons')
])
def update_tabs(df,path,file,matrix,sensor,datum,interval,initials):
df_block=pd.read_csv('G:/Shared drives/CZN_HydroGroup/Data/data_tables/block.csv')
block_start_time=df.index[0]
block_end_time=df.index[len(df.index)-1]
site_id=file.split('_')[0]
n=len(df['LEVEL'])-1
process_date=date.today().strftime('%m/%d/%Y')
quality='P'
sensor_sn=find_sn(path,file)
blockno=df_block['blockno'].max()+1
ind1=int(df_block['index2'].max()+1)
ind2=int(ind1+n)
dat_type='L'
unit='M'
df_block=df_block.append({'blockno':blockno,
'site_id':site_id,
'start_time':block_start_time,
'index1':ind1,
'end_time':block_end_time,
'index2':ind2,
'matrix':matrix,
'data_type':dat_type,
'sensor':sensor,
'sensor_sn':sensor_sn,
'unit':unit,
'interval':interval,
'mp_datum':datum,
'accuracy':acc,
'drift':np.nan,
'process_initials':initials,
'process_date':process_date,
'quality':quality},ignore_index=True)
df_data=pd.DataFrame()
df_data['index']=list(range(ind1,ind2+1,1))
df_data['blockno']=blockno
df_data['amount']=df['LEVEL_cor'].to_list()
#SC
dat_type='C'
unit='US'
blockno=df_block['blockno'].max()+1
ind1=int(df_block['index2'].max()+1)
ind2=int(ind1+n)
df_block=df_block.append({'blockno':blockno,
'site_id':site_id,
'start_time':block_start_time,
'index1':ind1,
'end_time':block_end_time,
'index2':ind2,
'matrix':matrix,
'data_type':dat_type,
'sensor':sensor,
'sensor_sn':sensor_sn,
'unit':unit,
'interval':interval,
'mp_datum':datum,
'accuracy':np.nan,
'drift':drift,
'process_initials':initials,
'process_date':process_date,
'quality':quality},ignore_index=True)
#df_data=pd.DataFrame()
df_app=pd.DataFrame()
df_app['index']=list(range(ind1,ind2+1,1))
df_app['blockno']=blockno
df_app['amount']=df['SC_cor'].to_list()
frames=[df_data,df_app]
df_data=pd.concat(frames)
#Temp
dat_type='T'
unit='DC'
blockno=df_block['blockno'].max()+1
ind1=int(df_block['index2'].max()+1)
ind2=int(ind1+n)
df_block=df_block.append({'blockno':blockno,
'site_id':site_id,
'start_time':block_start_time,
'index1':ind1,
'end_time':block_end_time,
'index2':ind2,
'matrix':matrix,
'data_type':dat_type,
'sensor':sensor,
'sensor_sn':sensor_sn,
'unit':unit,
'interval':interval,
'mp_datum':datum,
'accuracy':np.nan,
'drift':np.nan,
'process_initials':initials,
'process_date':process_date,
'quality':quality},ignore_index=True)
#df_data=pd.DataFrame()
df_app=pd.DataFrame()
df_app['index']=list(range(ind1,ind2+1,1))
df_app['blockno']=blockno
df_app['amount']=df['TEMPERATURE'].to_list()
frames=[df_data,df_app]
df_data=pd.concat(frames)
df_block.to_csv('G:/Shared drives/CZN_HydroGroup/Data/data_tables/block.csv',index=False)
df_data=df_data.to_csv('G:/Shared drives/CZN_HydroGroup/Data/data_tables/data.csv',mode='a',header=False,index=False)
#Move processed file to postprocess_files folder
df.to_csv(path+file)
postpath='G:/Shared drives/CZN_HydroGroup/Data/postprocess_files/'
os.replace(path+file,postpath+file)
def main(args=None):
cpu_number = multiprocessing.cpu_count()
parser = argparse.ArgumentParser(description='Path of networks')
parser.add_argument('-m', type=str, help='Multiplex')
args = parser.parse_args(args)
graph_path = args.m
########################################################################
# Parameters
########################################################################
EMBED_DIMENSION = 128
CLOSEST_NODES = np.int64(20)
NUM_SAMPLED = np.int64(3)
LEARNING_RATE = np.float64(0.01)
KL = False
NB_CHUNK = np.int64(1)
CHUNK_SIZE = np.int64(10)
NUM_STEPS_1 = np.int64(100 * 10**6 / CHUNK_SIZE)
graph_name = os.path.basename(graph_path)
###################################################################################"
# MULTIVERSE-M
###################################################################################"
r_readRDS = robjects.r['readRDS']
print('RWR-M')
proc = subprocess.Popen(['Rscript', './RWR/GenerateSimMatrix.R', \
'-n', "."+ args.m, '-o', \
'../ResultsRWR/MatrixSimilarityMultiplex'+graph_name, '-c',str(cpu_number)])
proc.wait()
pid = proc.pid
proc.kill()
print('RWR done')
r_DistancematrixPPI = r_readRDS('./ResultsRWR/MatrixSimilarityMultiplex' +
graph_name + '.rds')
gc.collect()
########################################################################
# Processing of the network
########################################################################
reverse_data_DistancematrixPPI, list_neighbours, nodes, data_DistancematrixPPI, nodes_incomponent, neighborhood, nodesstr \
= f.netpreprocess(r_DistancematrixPPI, graph_path, CLOSEST_NODES)
########################################################################
# Initialization
########################################################################
embeddings = np.random.normal(0, 1, [np.size(nodes), EMBED_DIMENSION])
########################################################################
# Training and saving best embeddings
########################################################################
nodes = np.asarray(nodes)
embeddings = f.train(neighborhood, nodes, list_neighbours, NUM_STEPS_1, NUM_SAMPLED, LEARNING_RATE, \
CLOSEST_NODES, CHUNK_SIZE, NB_CHUNK, embeddings, reverse_data_DistancematrixPPI)
X = dict(zip(range(embeddings.shape[0]), embeddings))
X = {str(int(nodesstr[key]) + 1): X[key] for key in X}
np.save('embeddings_M', X)
date = datetime.datetime.now()
os.replace('embeddings_M.npy', './ResultsMultiVERSE/' + 'embeddings_M.npy')
print('End')
if os.path.islink(filepath):
oldsymlinks.add(filepath)
for filename in filenames:
filepath = os.path.join(dirpath, filename)[2:].encode(
'ascii') # [2:] is to strip the ./ prefix
if not os.path.islink(filepath) and not filepath.startswith(
b'snapshotindex.txt') and not filepath.startswith(
b'raspbmirrordownloads.txt'):
basefiles.add(filepath)
print('stage 4, moves and deletions')
for filepath in fileupdates:
print((b'renaming ' + makenewpath(filepath) + b' to ' +
filepath).decode('ascii'))
os.replace(makenewpath(filepath), filepath)
for (filepath, symlinktarget) in symlinkupdates:
print('updating symlink ' + filepath.decode('ascii') + ' -> ' +
symlinktarget.decode('ascii'))
os.remove(filepath)
os.symlink(symlinktarget, filepath)
removedfiles = (basefiles | oldsymlinks) - (set(knownfiles.keys())
| newsymlinks)
def isemptydir(dirpath):
#scandir would be significantly more efficient, but needs python 3.6 or above
#which is not reasonable to expect at this time.
#return os.path.isdir(dirpath) and ((next(os.scandir(dirpath), None)) is None)
shutil.rmtree('output')
shutil.rmtree('output_2')
shutil.rmtree('output_3')
os.mkdir('output')
os.mkdir('output_2')
os.mkdir('output_3')
os.system('ffmpeg -i '+vidFile+' -f image2 "output/output_%06d.bmp"')
#
# FIRST CALL
#
for (path, dirnames, filenames) in os.walk('output'):
files.extend(os.path.join(name) for name in filenames)
for file in islice(files, 2, None, 3):
print('FIRST: '+file)
os.replace('output/'+file,'output_2/'+file)
files.clear()
time.sleep(2)
#
# SECOND CALL
#
if secondCall == True:
for (path, dirnames, filenames) in os.walk('output'):
files.extend(os.path.join(name) for name in filenames)
for file in islice(files, 2, None, 2):
print('FIRST: '+file)
os.replace('output/'+file,'output_2/'+file)
files.clear()
time.sleep(2)
#Number every 3rd (minus them)
elif data[begin + bnum:begin + bnum + 1] == b'\n':
result[data[begin:bnum + begin]] = b''
break
elif data[begin + bnum:begin + bnum + 1] == b',':
result[data[begin:bnum + begin]] = b''
break
else:
bnum = bnum + 1
for swap in list(result):
data = data.replace(swap, result[swap])
wflag = 1
else:
if args.status:
print('no')
sys.exit()
if wflag:
if args.backup:
from shutil import copyfile
if not args.status and existf(filename + '.bak') != 0:
print('Backing up ' + filename + ' ...')
copyfile(filename, filename + '.bak')
with open(filename + '_new', 'wb') as o:
devnull = o.write(data)
os.replace(filename + '_new', filename)
print('\n' + filename + ' patched\n')
else:
print('\nNo changes made to ' + filename + '\n')
