def record(self):
#Start recording:
self.status = 1
while self.status:
try:
received_line = self.get_message()[:-1] #strip \cr
temp = received_line.split(" ")
distance = int(temp[0][1:])
activeValue = int(temp[1][1:])
if distance > settings.DMAX:
activeValue = 0
else:
activeValue = 1
logging.info(received_line)
# Write to a temp file then rename to the final file,
# on mac, linux and unix this is atomic and on windows
# it's pretty fast
open(settings.OUPTUT_FILE + ".temp", "wb").write(json.dumps([{"active": activeValue}]))
shutil.move(settings.OUPTUT_FILE + ".temp", settings.OUPTUT_FILE)
if not settings.SILENT:
print "\nsensor data: " + received_line
print "Max distance: %s in." % (settings.DMAX,)
time.sleep(settings.RATE)
except ValueError:
if not settings.SILENT:
print "Serial error, skipping measurement"
except (KeyboardInterrupt, SystemExit):
self.cur.close()
self.db.close()
raise
def savefile(path, tagdata):
"""Saves tagdata to file at path."""
fp = open(path + ".new", "w")
pickle.dump(tagdata, fp)
fp.close()
shutil.move(path + ".new", path)
def __update_hotkey(self, command, hotkey):
""" Update the hotkey for 'command' to 'hotkey'. """
""" If 'command' is not found, add it with the new 'hotkey'. """
""" Return 'True' on success, 'False' otherwise. """
self.__touch_config_file()
oldfile = open(XBINDKEYS_CONFIG_FILE, "r")
newfile = open(XBINDKEYS_CONFIG_FILE + ".new", "w")
# Search for command
commandfound = False
skipnextline = False
for line in oldfile:
if skipnextline == False:
newfile.write(line)
else:
skipnextline = False
if line == '"' + command + '"\n':
newfile.write(" " + hotkey + "\n") # update hotkey
commandfound = True
skipnextline = True
if commandfound == False:
# command not found, add it
newfile.write('"' + command + '"\n')
newfile.write(" " + hotkey + "\n")
oldfile.close()
newfile.close()
try:
os.remove(XBINDKEYS_CONFIG_FILE)
except:
sessionlog.write("ERROR: 'Hotkeys.__update_hotkey()' - Cannot replace '" + XBINDKEYS_CONFIG_FILE + "'.")
os.remove(XBINDKEYS_CONFIG_FILE + ".new")
return False
shutil.move(XBINDKEYS_CONFIG_FILE + ".new", XBINDKEYS_CONFIG_FILE)
return True
def generateLocalConfig(localFile, build):
"""
Create resources/conf/localconfig.json and set the build values
appropriately.
"""
tmpOut = localFile + ".out"
with open(tmpOut, 'w') as f:
if os.path.exists(localFile):
data = json.load(open(localFile))
else:
data = {}
if build == 'debug-android' or build == 'debug-ios' or build == 'billingtest-android':
data['release'] = "debug"
elif build == 'ads-android' or build == 'ads-ios':
data['release'] = "ads"
elif build == 'paid-android' or build == 'paid-ios':
data['release'] = "paid"
elif build == 'samsung-android':
data['release'] = "samsung"
else:
assert 0, "run with build=something"
# force off the debug flag while building.
if data['release'] in ['paid', 'ads']:
data['debug'] = False
json.dump(data, f, indent=4)
shutil.move(tmpOut, localFile)
def __remove_hotkey(self, command):
""" Remove the hotkey for 'command' (and 'command' too, of course). """
""" Return 'True' on success, 'False' otherwise. """
self.__touch_config_file()
oldfile = open(XBINDKEYS_CONFIG_FILE, "r")
newfile = open(XBINDKEYS_CONFIG_FILE + ".new", "w")
commandfound = False
skipnextline = False
for line in oldfile:
if skipnextline != True:
if line != '"' + command + '"\n':
newfile.write(line)
else:
commandfound = True
skipnextline = True
else:
skipnextline = False
oldfile.close()
newfile.close()
if commandfound == True:
try:
os.remove(XBINDKEYS_CONFIG_FILE)
except:
sessionlog.write("ERROR: 'Hotkeys.__remove_hotkey()' - Cannot replace '" + XBINDKEYS_CONFIG_FILE + "'.")
os.remove(XBINDKEYS_CONFIG_FILE + ".new")
return False
shutil.move(XBINDKEYS_CONFIG_FILE + ".new", XBINDKEYS_CONFIG_FILE)
else:
os.remove(XBINDKEYS_CONFIG_FILE + ".new")
return True
def get(url, path, verbose=False):
sha_url = url + ".sha256"
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
temp_path = temp_file.name
with tempfile.NamedTemporaryFile(suffix=".sha256", delete=False) as sha_file:
sha_path = sha_file.name
try:
download(sha_path, sha_url, False, verbose)
if os.path.exists(path):
if verify(path, sha_path, False):
if verbose:
print("using already-download file " + path)
return
else:
if verbose:
print("ignoring already-download file " + path + " due to failed verification")
os.unlink(path)
download(temp_path, url, True, verbose)
if not verify(temp_path, sha_path, verbose):
raise RuntimeError("failed verification")
if verbose:
print("moving {} to {}".format(temp_path, path))
shutil.move(temp_path, path)
finally:
delete_if_present(sha_path, verbose)
delete_if_present(temp_path, verbose)
def docss():
""" Compresses the CSS files """
listCSS = []
theme = settings.get_theme()
print "Using theme %s" % theme
css_cfg = os.path.join("..", "..", "..", "private", "templates", theme, "css.cfg")
f = open(css_cfg, "r")
files = f.readlines()
f.close()
for file in files[:-1]:
p = re.compile("(\n|\r|\t|\f|\v)+")
file = p.sub("", file)
listCSS.append("../../styles/%s" % file)
outputFilenameCSS = "eden.min.css"
# Merge CSS files
print "Merging Core styles."
mergedCSS = mergeCSS(listCSS, outputFilenameCSS)
# Compress CSS files
print "Writing to %s." % outputFilenameCSS
compressCSS(mergedCSS, outputFilenameCSS)
# Move files to correct locations
print "Deleting %s." % outputFilenameCSS
try:
os.remove("../../themes/%s/%s" % (theme, outputFilenameCSS))
except:
pass
print "Moving new %s." % outputFilenameCSS
shutil.move(outputFilenameCSS, "../../themes/%s" % theme)
def _fetch_pkg(self, gopath, pkg, rev):
"""Fetch the package and setup symlinks."""
fetcher = self._get_fetcher(pkg)
root = fetcher.root()
root_dir = os.path.join(self.workdir, 'fetches', root, rev)
# Only fetch each remote root once.
if not os.path.exists(root_dir):
with temporary_dir() as tmp_fetch_root:
fetcher.fetch(dest=tmp_fetch_root, rev=rev)
safe_mkdir(root_dir)
for path in os.listdir(tmp_fetch_root):
shutil.move(os.path.join(tmp_fetch_root, path), os.path.join(root_dir, path))
# TODO(John Sirois): Circle back and get get rid of this symlink tree.
# GoWorkspaceTask will further symlink a single package from the tree below into a
# target's workspace when it could just be linking from the fetch_dir. The only thing
# standing in the way is a determination of what we want to artifact cache. If we don't
# want to cache fetched zips, linking straight from the fetch_dir works simply. Otherwise
# thought needs to be applied to using the artifact cache directly or synthesizing a
# canonical owner target for the fetched files that 'child' targets (subpackages) can
# depend on and share the fetch from.
dest_dir = os.path.join(gopath, 'src', root)
# We may have been `invalidate`d and not `clean-all`ed so we need a new empty symlink
# chroot to avoid collision; thus `clean=True`.
safe_mkdir(dest_dir, clean=True)
for path in os.listdir(root_dir):
os.symlink(os.path.join(root_dir, path), os.path.join(dest_dir, path))
def configure_linter(self, language):
"""Fill out the template and move the linter into Packages."""
try:
if language is None:
return
if not self.fill_template(self.temp_dir, self.name, self.fullname, language):
return
git = util.which('git')
if git:
subprocess.call((git, 'init', self.temp_dest))
shutil.move(self.temp_dest, self.dest)
util.open_directory(self.dest)
self.wait_for_open(self.dest)
except Exception as ex:
sublime.error_message('An error occurred while configuring the plugin: {}'.format(str(ex)))
finally:
if self.temp_dir and os.path.exists(self.temp_dir):
shutil.rmtree(self.temp_dir)
def moveOutput_VRay(arg,frameNr, verbose):
if ((not arg.ElementsFolder) or (not arg.vraySeperateRenderChannels)):
return
import shutil
VRayExt=arg.FExt
if (arg.vraySeperateRenderChannels):
VRayExt=os.path.splitext(MaxPlus.Core.EvalMAXScript("renderers.production.output_splitfilename").Get())[1]
if (len(VRayExt)==0):
VRayExt=arg.FExt
FNameVar_Rendered= arg.FNameVar;
FNameVar_Rendered= FNameVar_Rendered.replace("<Channel>\\","")
if (argValid(arg.FNameVar) and MaxPlus.Core.EvalMAXScript("(maxOps.GetCurRenderElementMgr()).GetElementsActive()").Get()):
nrOfElements=MaxPlus.Core.EvalMAXScript("(maxOps.GetCurRenderElementMgr()).NumRenderElements()").Get()
for elemNr in xrange(0,nrOfElements):
elemName=MaxPlus.Core.EvalMAXScript("((maxOps.GetCurRenderElementMgr()).GetRenderElement "+str(elemNr)+").elementName").Get()
elemName=elemName.replace(" ","_")
fileout_Rendered=FNameVar_Rendered+str(frameNr).zfill(int(arg.FPadding))+VRayExt
fileout_Should =arg.FNameVar +str(frameNr).zfill(int(arg.FPadding))+VRayExt
fileout_Rendered=fileout_Rendered.replace("<Channel>",elemName)
fileout_Should =fileout_Should .replace("<Channel>",elemName)
if (not os.path.isfile(fileout_Rendered)):
logMessageGen("WRN", "Element file to be moved to subfolder not found: "+fileout_Rendered,1)
continue;
if (verbose):
logMessage("Moving element '"+fileout_Rendered+"' => '"+fileout_Should+"'")
shutil.move(fileout_Rendered,fileout_Should)
def extract_zip(self, filename):
with zipfile.ZipFile(filename) as f:
# extractall doesn't work if zipfiles do not include directories
# as separate members (as for github and bitbucket)
topdirs = []
for m in f.namelist():
if m.endswith("/"):
if len(m.split("/")) == 2:
topdirs.append(m.split("/")[0])
continue
directory = "/".join(m.split("/")[0:-2])
if directory.find("/") == -1:
topdirs.append(directory)
if (len(directory) > 0 and
not directory.startswith("/") and
(directory.find("..") == -1) and
not os.path.exists(directory)):
os.makedirs(directory)
sys.stdout.write(".")
f.extract(m)
topdirs = filter(lambda x: len(x) != 0, topdirs)
topdirs = list(set(topdirs))
if not topdirs:
topdirs = [f.namelist()[0]]
if len(topdirs) > 1:
os.makedirs(self.path)
for d in topdirs:
shutil.move(d, "%s/%s" % (self.path, d))
else:
shutil.move(topdirs[0], self.path)
def gen_moc():
workDir = os.getcwd() + "/Gui"
print workDir
dirList = os.listdir(workDir)
for files in dirList:
if files.endswith(".h"):
print files
outp = "moc_" + files[:-2] + ".cpp"
print outp
affected = "./Gui/" + files
call(["moc", affected, "-o", outp])
#os.rename(outp, "./Gui/" + outp)
l = -1
f = open(outp)
lines = f.readlines()
f.close()
for line in lines:
if line[0] == '#':
break;
l = l + 1
if l >= 0:
f = open(outp, 'w');
lines[l] = '\n#include "EdPrec.h"\n'
f.writelines(lines)
f.close()
shutil.move(outp, "./Gui/" + outp)
def filter_and_persist_proprietary_tool_panel_configs( self, tool_configs_to_filter ):
"""Eliminate all entries in all non-shed-related tool panel configs for all tool config file names in the received tool_configs_to_filter."""
for proprietary_tool_conf in self.proprietary_tool_confs:
persist_required = False
tree, error_message = xml_util.parse_xml( proprietary_tool_conf )
if tree:
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
# Tools outside of sections.
file_path = elem.get( 'file', None )
if file_path:
if file_path in tool_configs_to_filter:
root.remove( elem )
persist_required = True
elif elem.tag == 'section':
# Tools contained in a section.
for section_elem in elem:
if section_elem.tag == 'tool':
file_path = section_elem.get( 'file', None )
if file_path:
if file_path in tool_configs_to_filter:
elem.remove( section_elem )
persist_required = True
if persist_required:
fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-toolshed-fapptpc" )
tmp_filename = fh.name
fh.close()
fh = open( tmp_filename, 'wb' )
tree.write( tmp_filename, encoding='utf-8', xml_declaration=True )
fh.close()
shutil.move( tmp_filename, os.path.abspath( proprietary_tool_conf ) )
os.chmod( proprietary_tool_conf, 0644 )
def env_teardown():
'''
Teardown the testing environment.
'''
print('\n-----DESTROY ENV')
if len(PATHS) == 1:
print('Env setup not completed, check network connection.')
return
src_log = [path for path in PATHS if '.log' in path][-1]
dst_log = os.path.join('/tmp', 'test.log')
try:
delete_it(dst_log)
shutil.move(src_log, dst_log)
except IOError:
pass
env_status()
for path in PATHS:
delete_it(path)
env_config_teardown()
print('\n-----DESTROY ENV FINISHED')
def _stage_final_image(self):
try:
fs_related.makedirs(self.__ensure_isodir() + "/LiveOS")
minimal_size = self._resparse()
if not self.skip_minimize:
fs_related.create_image_minimizer(self.__isodir + \
"/LiveOS/osmin.img",
self._image,
minimal_size)
if self.skip_compression:
shutil.move(self._image, self.__isodir + "/LiveOS/ext3fs.img")
else:
fs_related.makedirs(os.path.join(
os.path.dirname(self._image),
"LiveOS"))
shutil.move(self._image,
os.path.join(os.path.dirname(self._image),
"LiveOS", "ext3fs.img"))
fs_related.mksquashfs(os.path.dirname(self._image),
self.__isodir + "/LiveOS/squashfs.img")
self.__create_iso(self.__isodir)
if self.pack_to:
isoimg = os.path.join(self._outdir, self.name + ".iso")
packimg = os.path.join(self._outdir, self.pack_to)
misc.packing(packimg, isoimg)
os.unlink(isoimg)
finally:
shutil.rmtree(self.__isodir, ignore_errors = True)
self.__isodir = None
def _stage_final_image(self):
try:
makedirs(self.__ensure_isodir() + "/LiveOS")
self._resparse()
if not self.skip_minimize:
create_image_minimizer(self.__isodir + "/LiveOS/osmin.img", self._image, self.compress_type)
if self.skip_compression:
shutil.move(self._image, self.__isodir + "/LiveOS/ext3fs.img")
if os.stat(self.__isodir + "/LiveOS/ext3fs.img").st_size >= 4*1024*1024*1024:
self._isofstype = "udf"
logging.warn("Switching to UDF due to size of LiveOS/ext3fs.img")
else:
makedirs(os.path.join(os.path.dirname(self._image), "LiveOS"))
shutil.move(self._image,
os.path.join(os.path.dirname(self._image),
"LiveOS", "ext3fs.img"))
mksquashfs(os.path.dirname(self._image),
self.__isodir + "/LiveOS/squashfs.img",
self.compress_type)
if os.stat(self.__isodir + "/LiveOS/squashfs.img").st_size >= 4*1024*1024*1024:
self._isofstype = "udf"
logging.warn("Switching to UDF due to size of LiveOS/squashfs.img")
self.__create_iso(self.__isodir)
finally:
shutil.rmtree(self.__isodir, ignore_errors = True)
self.__isodir = None
def hadoop_jar(stdout, stderr, environ, *args):
if len(args) < 1:
stderr.write('RunJar jarFile [mainClass] args...\n')
return -1
jar_path = args[0]
if not os.path.exists(jar_path):
stderr.write(
'Exception in thread "main" java.io.IOException: Error opening job'
' jar: %s\n' % jar_path)
return -1
# only simulate for streaming steps
if HADOOP_STREAMING_JAR_RE.match(os.path.basename(jar_path)):
streaming_args = args[1:]
output_idx = list(streaming_args).index('-output')
assert output_idx != -1
output_dir = streaming_args[output_idx + 1]
real_output_dir = hdfs_path_to_real_path(output_dir, environ)
mock_output_dir = get_mock_hadoop_output()
if mock_output_dir is None:
stderr.write('Job failed!')
return -1
if os.path.isdir(real_output_dir):
os.rmdir(real_output_dir)
shutil.move(mock_output_dir, real_output_dir)
now = datetime.datetime.now()
stderr.write(now.strftime('Running job: job_%Y%m%d%H%M_0001\n'))
stderr.write('Job succeeded!\n')
return 0
def _move_additional_files(self, old_filename, new_filename):
"""Move extra files, like playlists..."""
old_path = encode_filename(os.path.dirname(old_filename))
new_path = encode_filename(os.path.dirname(new_filename))
patterns = encode_filename(config.setting["move_additional_files_pattern"])
patterns = filter(bool, [p.strip() for p in patterns.split()])
try:
names = os.listdir(old_path)
except os.error:
log.error("Error: {} directory not found".format(old_path))
return
filtered_names = filter(lambda x: x[0] != '.', names)
for pattern in patterns:
pattern_regex = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
file_names = names
if pattern[0] != '.':
file_names = filtered_names
for old_file in file_names:
if pattern_regex.match(old_file):
new_file = os.path.join(new_path, old_file)
old_file = os.path.join(old_path, old_file)
# FIXME we shouldn't do this from a thread!
if self.tagger.files.get(decode_filename(old_file)):
log.debug("File loaded in the tagger, not moving %r", old_file)
continue
log.debug("Moving %r to %r", old_file, new_file)
shutil.move(old_file, new_file)
def __renameTopperFileStage1 (self):
"""
Rotate (move) Tooper Records File To Temp File
When All Nginx Processes close the file move it to final File in stage2
"""
dataFileName = "data" + str('%08d' % self.__topperFileNum) + ".data.tmp"
newFileName = os.path.join(self.__conf.topperDir, self.__conf.kConf.kTopperDataDir, dataFileName)
try:
shutil.move(self.__conf.actualNgxRecordsLogFile,newFileName)
retVal = True
self.__log("rename-topper-base-file").debug3("Rename Topper File (Rotation) = %s to Temp File = %s",self.__conf.actualNgxRecordsLogFile,dataFileName)
except:
self.__log("rename-topper-base-file-fail").error("Rename Topper File (Rotation) = %s to Temp File = %s Failed",self.__conf.actualNgxRecordsLogFile, dataFileName)
retVal = False
if retVal:
self.__topperTempFile = newFileName
self.__topperFileNum = self.__topperFileNum + 1
# This is because '%08'
self.__topperFileName = self.__topperFileNum % 100000000
else:
self.__topperTempFile = ""
return retVal
def _initialize_metadata(self):
self._logger.info("Initializing the file metadata for {}...".format(self.basefolder))
old_metadata_path = os.path.join(self.basefolder, "metadata.yaml")
backup_path = os.path.join(self.basefolder, "metadata.yaml.backup")
if os.path.exists(old_metadata_path):
# load the old metadata file
try:
with open(old_metadata_path) as f:
import yaml
self._old_metadata = yaml.safe_load(f)
except:
self._logger.exception("Error while loading old metadata file")
# make sure the metadata is initialized as far as possible
self._list_folder(self.basefolder)
# rename the old metadata file
self._old_metadata = None
try:
import shutil
shutil.move(old_metadata_path, backup_path)
except:
self._logger.exception("Could not rename old metadata.yaml file")
else:
# make sure the metadata is initialized as far as possible
self._list_folder(self.basefolder)
self._logger.info("... file metadata for {} initialized successfully.".format(self.basefolder))
def _move_into_dir(self, file, dir):
if not self._dryrun and not os.path.exists(dir):
os.makedirs(dir)
filename = os.path.basename(file)
if not self._dryrun:
shutil.move(file, os.path.join(dir, filename))
print file, " => ", dir
def cleanup(self, action):
if not self.steps_filename:
return
if not self.question_yes_no("All unused PPM files will be moved to a"
" backup directory. Are you sure?",
"Clean up data directory?"):
return
# Remember the current step index
current_step_index = self.current_step_index
# Get the backup dir
backup_dir = os.path.join(self.steps_data_dir, "backup")
# Create it if it doesn't exist
if not os.path.exists(backup_dir):
os.makedirs(backup_dir)
# Move all files to the backup dir
for filename in glob.glob(os.path.join(self.steps_data_dir,
"*.[Pp][Pp][Mm]")):
shutil.move(filename, backup_dir)
# Get the used files back
for step in self.steps:
self.set_state_from_step_lines(step, backup_dir, warn=False)
self.get_step_lines(self.steps_data_dir)
# Remove the used files from the backup dir
used_files = os.listdir(self.steps_data_dir)
for filename in os.listdir(backup_dir):
if filename in used_files:
os.unlink(os.path.join(backup_dir, filename))
# Restore step index
self.set_step(current_step_index)
# Inform the user
self.message("All unused PPM files may be found at %s." %
os.path.abspath(backup_dir),
"Clean up data directory")
def _put_filename(self, key, filename):
target = self._build_filename(key)
shutil.move(filename, target)
# we do not know the permissions of the source file, rectify
self._fix_permissions(target)
return key
def make_target_directory(self, path):
path = os.path.abspath(path)
try:
os.makedirs(path)
except OSError as e:
self.abort('Could not create target folder: %s' % e)
if os.path.isdir(path):
try:
if len(os.listdir(path)) != 0:
raise OSError('Directory not empty')
except OSError as e:
self.abort('Bad target folder: %s' % e)
scratch = os.path.join(tempfile.gettempdir(), uuid.uuid4().hex)
os.makedirs(scratch)
try:
yield scratch
except:
shutil.rmtree(scratch)
raise
else:
# Use shutil.move here in case we move across a file system
# boundary.
for filename in os.listdir(scratch):
if isinstance(path, unicode):
filename = filename.decode(fs_enc)
shutil.move(os.path.join(scratch, filename),
os.path.join(path, filename))
os.rmdir(scratch)
def initialize(self, test, log):
'''Does the init part of the test
1.Finds initial count of entry in log
2.Creates a file 'cron' under cron.d
3.Backs up /etc/crontab
4.Modifies /etc/crontab '''
self.log = log
self.initial_count = self.count_log('Cron automation')
f = open('/etc/cron.d/cron', 'w')
f.write('''#!/bin/bash
touch %s
echo 'Cron automation' >> %s
''' % (self.log, self.log))
f.close()
utils.system('chmod +x /etc/cron.d/cron')
shutil.copyfile('/etc/crontab', '/tmp/backup')
f = open('/etc/crontab', 'w')
f.write('* * * * * root run-parts /etc/cron.d/\n')
f.close()
if test == 'deny_cron':
if os.path.exists('/etc/cron.d/jobs.deny'):
shutil.move('/etc/cron.d/jobs.deny', '/tmp/jobs.deny')
f = open('/etc/cron.d/jobs.deny', 'w')
f.write('cron')
f.close()
elif test == 'allow_cron' :
os.remove('/etc/cron.d/jobs.deny')
if os.path.exists('/etc/cron.d/jobs.allow'):
shutil.move('/etc/cron.d/jobs.allow', '/tmp/jobs.allow')
f = open('/etc/cron.d/jobs.allow', 'w')
f.write('cron')
f.close()
def forwards(self, orm):
"Write your forwards migration here"
for item in orm.AlbumConvertableItem.objects.all():
try:
image_path = item.image.image.path
except:
image_path = os.path.join(settings.MEDIA_ROOT, item.thumbFilename)
try:
os.stat(image_path)
except OSError as e:
if e.errno != 2:
raise e
else:
continue
old_dir, filename = os.path.split(image_path)
new_path = os.path.join('albums', str(item.parent.pk), str(item.pk))
new_dir = os.path.join(settings.MEDIA_ROOT, new_path)
try:
os.makedirs(new_dir)
except OSError as e:
if(e.errno != 17):
raise e
print "Directory %s already exists" % new_dir
print "Moving %s" % image_path
if(image_path != os.path.join(new_dir, filename)):
shutil.move(image_path, new_dir)
else:
print "Skipping"
item.preview = os.path.join(new_path, filename)
item.save()
def deploy_wnmp():
os.chdir(os.path.join(BASE_DIR, 'wnmp'))
git_export('wnmp', TARGET_DIR)
# PHP
wget('http://windows.php.net/downloads/releases/'
'php-5.4.5-Win32-VC9-x86.zip',
sha1='028eb12e09fe011e20097c82064d6c550bf896c4')
logging.info('Extracting PHP...')
path = os.path.join(BASE_DIR, '_tmp', 'php')
makedirs(path, exist_ok=True)
ar = zipfile.ZipFile(
os.path.join(BASE_DIR, 'php-5.4.5-Win32-VC9-x86.zip'))
ar.extractall(path)
shutil.rmtree(os.path.join(TARGET_DIR, 'php'))
shutil.copytree(path, os.path.join(TARGET_DIR, 'php'))
# nginx
wget('http://nginx.org/download/nginx-1.2.2.zip',
sha1='0a5dfbb766bfefa238207db25d7b64b69aa37908')
logging.info('Extracting nginx...')
path = os.path.join(BASE_DIR, '_tmp')
makedirs(path, exist_ok=True)
ar = zipfile.ZipFile(
os.path.join(BASE_DIR, 'nginx-1.2.2.zip'))
ar.extractall(path)
shutil.rmtree(os.path.join(TARGET_DIR, 'nginx'))
shutil.copytree(os.path.join(path, 'nginx-1.2.2'),
os.path.join(TARGET_DIR, 'nginx'))
shutil.move(os.path.join(TARGET_DIR, 'example.nginx.conf'),
os.path.join(TARGET_DIR, 'nginx', 'conf', 'nginx.conf'))
# cleanup
shutil.rmtree(os.path.join(BASE_DIR, '_tmp'))
def main(max_stations=0, folder='.'):
try:
makedirs(output_folder+'/'+folder)
except OSError:
pass
all_files = [ f for f in listdir(data_folder) if isfile(join(data_folder,f)) and f.endswith('.gz') ]
for ndf in all_files:
string = '_%dstations' % max_stations
new_name=ndf[:-7]+string+ndf[-7:]
rename(data_folder+'/'+ndf, data_folder+'/'+new_name)
all_files = [ f for f in listdir(data_folder) if isfile(join(data_folder,f)) and f.endswith('.gz') ]
for a_f in all_files:
move(data_folder+'/'+a_f, output_folder+'/'+folder+'/'+a_f)
print "Moved:", a_f[0:-3]
data_files = [ f for f in listdir(output_folder+'/'+folder) if isfile(join(output_folder+'/'+folder,f)) and f.endswith('.dat.gz') ]
print "\n"
for d_f in data_files:
fin = gzip.open(output_folder+'/'+folder+'/'+d_f, 'rb')
data = fin.read()
fin.close()
with open(output_folder+'/'+folder+'/'+d_f[0:-3],'w') as fout:
fout.write(data)
print "Unzipped:", d_f[0:-3]
def docss():
""" Compresses the CSS files """
listCSS = []
f = open("sahana.css.cfg", "r")
files = f.readlines()
f.close()
for file in files[:-1]:
p = re.compile("(\n|\r|\t|\f|\v)+")
file = p.sub("", file)
listCSS.append("../../styles/%s" % file)
outputFilenameCSS = "sahana.min.css"
# Merge CSS files
print "Merging Core styles."
mergedCSS = mergeCSS(listCSS, outputFilenameCSS)
# Compress CSS files
print "Writing to %s." % outputFilenameCSS
compressCSS(mergedCSS, outputFilenameCSS)
# Move files to correct locations
print "Deleting %s." % outputFilenameCSS
try:
os.remove("../../styles/S3/%s" % outputFilenameCSS)
except:
pass
print "Moving new %s." % outputFilenameCSS
shutil.move(outputFilenameCSS, "../../styles/S3")
def load(self, fmu_path, output_dir):
"""Load the FMU for continued simulation in :meth:`continue_run`.
"""
import pyfmi
if 'log_level' in self._options:
self.fmu = pyfmi.load_fmu(fmu_path, log_level=self.log_level)
else:
self.fmu = pyfmi.load_fmu(fmu_path)
# Initialize the fmu, only call setup_experiment for FMUs 2.0
try:
self.fmu.setup_experiment()
version = 2
except AttributeError:
version = 1
self.fmu.initialize()
# Copy the log file to the result directory
log = ''
if version == 1:
log = self.fmu.get_identifier()
if version == 2:
log = self.fmu.get_name()
log += '_log.txt'
source = os.path.join(os.getcwd(), log)
destination = os.path.join(output_dir, 'log%i.txt' % self.interval)
move(source, destination)
def move_files(top_level_path):
for folder in os.listdir(top_level_path):
for file in os.listdir(os.path.join(top_level_path, folder)):
print(file)
shutil.move(os.path.join(top_level_path, folder, file), os.path.join(top_level_path, file))
def gestion_importation_sap(event):
try:
if event.wd == 1:
if event.name.endswith(".csv") or event.name.endswith(".CSV"):
log_print("Importation " + event.pathname)
err = stock_labo.models.import_navette_sap(
emplacement=event.pathname,
nom="",
extension="",
fin_de_ligne="\r\n")
if err == "":
multi_file_backups(event.path + "/backup/",
event.name[:-4] + ".ok")
shutil.move(
event.pathname,
event.path + "/backup/" + event.name[:-4] + ".ok")
else:
multi_file_backups(event.path + "/error/",
event.name[:-4] + ".err")
multi_file_backups(event.path + "/error/",
event.name[:-4] + ".log")
shutil.move(
event.pathname,
event.path + "/error/" + event.name[:-4] + ".err")
ferr = codecs.open(event.path + "/error/" +
event.name[:-4] + ".log",
"w",
encoding="utf-8")
ferr.writelines(err)
ferr.close()
mail_corps = "Le fichier %s a provoqué une erreur : \n\n" % (
event.name)
ferr = codecs.open(event.path + "/error/" +
event.name[:-4] + ".err",
"r",
encoding="utf-8")
mail_corps += ferr.readline()
ferr.close()
mail_corps += "\n\n"
mail_corps = "Le fichier %s a provoqué une erreur : \n\n" % (
event.name)
ferr = codecs.open(event.path + "/error/" +
event.name[:-4] + ".log",
"r",
encoding="utf-8")
mail_corps += ferr.readline()
ferr.close()
m = mail.Mail(mail_corps)
m.from_("*****@*****.**")
m.reply_to("*****@*****.**")
#m.to(["*****@*****.**","*****@*****.**"])
m.to(
["*****@*****.**", "*****@*****.**"])
if version.status_developement["data"] == "prod":
m.subject(
"Stock Labo Paris, Erreur importation fichier navette SAP"
)
else:
m.subject(
"Ne pas tenir compte de cette email, je fait des tests sur le serveur, charly"
)
#m.send("smtp.laposte.net", "*****@*****.**", "charly.gontero", "1Addyson")
m.send(smtp="par-srv-cas01.eu.takasago.com",
expediteur="*****@*****.**",
user=None,
password=None,
starttls=False)
connection_db_ouverte = True
else:
shutil.move(event.pathname,
event.path + "/error/" + event.name)
log_print(
"rien n'a faire, le fichier n'a pas la bonne extention : %s"
% (event.name))
else:
log_print("rien n'a faire, événement d'un dossier fils wd : %d" %
(event.wd))
except:
tb = traceback.format_exc()
log_print("!!!!! ERREUR !!!!!")
log_print(tb)
log_print("!!!!! ERREUR !!!!!\n\n")
raise ImportationErreur
def generate(self, should_notify=True):
"""
Generate the app
:return: the path to the generated apk
"""
self.update_status('Preparing parent source code')
self.prepare_source()
self.app_package_name = 'org.fossasia.openevent.' + re.sub('\W+', '', self.app_name)
config = {
'Email': self.creator_email,
'App_Name': self.app_name,
'Api_Link': self.api_link
}
self.update_status('Generating app configuration')
with open(self.get_path("app/src/main/assets/config.json"), "w+") as config_file:
config_file.write(json.dumps(config))
self.update_status('Generating launcher icons & background image')
resize_launcher_icon(self.app_launcher_icon, self.app_working_dir)
resize_background_image(self.app_background_image, self.app_working_dir)
self.update_status('Updating resources')
replace(self.get_path("app/src/main/res/values/strings.xml"), 'OpenEvent', self.app_name)
replace(self.get_path("app/src/main/res/layout/nav_header.xml"), 'twitter', 'background')
replace(self.get_path("app/build.gradle"), '"org.fossasia.openevent"', '"%s"' % self.app_package_name)
self.update_status('Loading assets')
for f in os.listdir(self.app_temp_assets):
path = os.path.join(self.app_temp_assets, f)
if os.path.isfile(path):
shutil.copyfile(path, self.get_path("app/src/main/assets/" + f))
self.update_status('Preparing android build tools')
build_tools_version = get_build_tools_version(self.get_path('app/build.gradle'))
build_tools_path = os.path.abspath(os.environ.get('ANDROID_HOME') + '/build-tools/' + build_tools_version)
self.update_status('Building android application package')
self.run_command([os.path.abspath(self.config['BASE_DIR'] + '/scripts/build_apk.sh'), build_tools_path])
self.update_status('Application package generated')
self.apk_path = self.get_path('release.apk')
if should_notify:
self.notify()
apk_url = '/static/releases/%s.apk' % self.identifier
shutil.move(self.apk_path, os.path.abspath(self.config['BASE_DIR'] + '/app/' + apk_url))
self.update_status('SUCCESS', message=apk_url)
self.cleanup()
return apk_url
def backup_DB():
os.makedirs("backup", exist_ok=True)
shutil.move("justice.db", "backup/justice.db")
def main(argv=None,
experiment_name="",
run_id=0,
csv_filename="gpt2_parity_results.csv"):
result = {}
from transformers import __version__ as transformers_version
if version.parse(transformers_version) < version.parse(
"3.1.0"): # past_key_values name does not exist in 3.0.2 or older
raise RuntimeError("This tool requires transformers 3.1.0 or later.")
args = parse_arguments(argv)
setup_logger(args.verbose)
if not experiment_name:
import sys
experiment_name = " ".join(argv if argv else sys.argv[1:])
if args.tolerance == 0:
args.tolerance = DEFAULT_TOLERANCE[args.precision]
logger.info(f"Arguments:{args}")
cache_dir = args.cache_dir
output_dir = args.output if not args.output.endswith(
".onnx") else os.path.dirname(args.output)
prepare_environment(cache_dir, output_dir, args.use_gpu)
if args.precision != Precision.FLOAT32:
assert args.optimize_onnx, "fp16/int8 requires --optimize_onnx"
if args.precision == Precision.FLOAT16:
assert args.use_gpu, "fp16 requires --use_gpu"
if args.precision == Precision.INT8:
assert not args.use_gpu, "quantization only supports CPU"
model_class = MODEL_CLASSES[args.model_class][0]
use_padding = MODEL_CLASSES[args.model_class][2]
if args.model_class == "GPT2LMHeadModel_BeamSearchStep":
model_type = "beam_search_step"
elif args.model_class == "GPT2LMHeadModel_ConfigurableOneStepSearch":
model_type = "configurable_one_step_search"
else:
model_type = "default"
gpt2helper = Gpt2HelperFactory.create_helper(model_type)
gpt2tester = Gpt2TesterFactory.create_tester(model_type)
config = AutoConfig.from_pretrained(args.model_name_or_path,
cache_dir=cache_dir)
if model_type == "beam_search_step":
model = model_class.from_pretrained(
args.model_name_or_path,
config=config,
batch_size=1,
beam_size=args.beam_size,
cache_dir=cache_dir,
)
elif model_type == "configurable_one_step_search":
model = model_class.from_pretrained(
args.model_name_or_path,
config=config,
batch_size=1,
beam_size=args.beam_size,
ignore_eos=args.ignore_eos,
temperature=args.temperature,
repetition_penalty=args.repetition_penalty,
excluded_token_ids=args.excluded_token_ids,
length_penalty=args.length_penalty,
do_sample=args.do_sample,
do_sample_top_p=args.do_sample_top_p,
do_sample_top_k=args.do_sample_top_k,
cache_dir=cache_dir,
)
else:
model = model_class.from_pretrained(args.model_name_or_path,
config=config,
cache_dir=cache_dir)
device = torch.device("cuda:0" if args.use_gpu else "cpu")
model.eval().to(device)
if (not args.use_external_data_format) and (config.n_layer > 24):
logger.info(f"Try --use_external_data_format when model size > 2GB")
onnx_model_paths = gpt2helper.get_onnx_paths(
output_dir,
args.model_name_or_path,
args.model_class,
new_folder=(args.precision == Precision.INT8),
remove_existing=["fp32", "fp16", "int8"],
) # Do not remove raw model to save time in parity test
raw_onnx_model = onnx_model_paths["raw"]
if os.path.exists(raw_onnx_model) and not args.overwrite:
logger.warning(
f"Skip exporting ONNX model since it existed: {raw_onnx_model}")
else:
logger.info(f"Exporting ONNX model to {raw_onnx_model}")
gpt2helper.export_onnx(
model,
device,
raw_onnx_model,
args.verbose,
args.use_external_data_format,
has_position_ids=use_padding,
has_attention_mask=use_padding,
input_ids_dtype=torch.int32
if args.use_int32_inputs else torch.int64,
position_ids_dtype=torch.int32
if args.use_int32_inputs else torch.int64,
attention_mask_dtype=torch.int32
if args.use_int32_inputs else torch.int64,
)
fp16_params = {"keep_io_types": args.keep_io_types}
if args.io_block_list:
fp16_params["keep_io_types"] = args.io_block_list
if args.node_block_list:
fp16_params["node_block_list"] = args.node_block_list
if args.op_block_list:
fp16_params["op_block_list"] = args.op_block_list
if args.force_fp16_initializers:
fp16_params["force_fp16_initializers"] = args.force_fp16_initializers
is_io_float16 = args.precision == Precision.FLOAT16 and not args.keep_io_types
if args.optimize_onnx or args.precision != Precision.FLOAT32:
output_path = onnx_model_paths[str(args.precision) if args.
precision != Precision.INT8 else "fp32"]
logger.info(f"Optimizing model to {output_path}")
gpt2helper.optimize_onnx(
raw_onnx_model,
output_path,
args.precision == Precision.FLOAT16,
model.config.num_attention_heads,
model.config.hidden_size,
args.use_external_data_format,
auto_mixed_precision=args.auto_mixed_precision,
**fp16_params,
)
else:
output_path = raw_onnx_model
if args.precision == Precision.INT8:
logger.info("quantizing model...")
QuantizeHelper.quantize_onnx_model(output_path,
onnx_model_paths["int8"],
args.use_external_data_format)
model = QuantizeHelper.quantize_torch_model(model)
logger.info("finished quantizing model")
output_path = onnx_model_paths["int8"]
if args.output.endswith(
".onnx"
) and output_path != args.output and not args.use_external_data_format:
import shutil
shutil.move(output_path, args.output)
output_path = args.output
logger.info(f"Output path: {output_path}")
model_size_in_MB = int(
get_onnx_model_size(output_path, args.use_external_data_format) /
1024 / 1024)
session = create_onnxruntime_session(output_path,
args.use_gpu,
enable_all_optimization=True,
verbose=args.verbose)
if args.model_class == "GPT2LMHeadModel" and session is not None:
parity_result = gpt2helper.test_parity(
session,
model,
device,
is_io_float16,
rtol=args.tolerance,
atol=args.tolerance,
model_class=args.model_class,
has_position_ids=use_padding,
has_attention_mask=use_padding,
input_ids_dtype=torch.int32
if args.use_int32_inputs else torch.int64,
position_ids_dtype=torch.int32
if args.use_int32_inputs else torch.int64,
attention_mask_dtype=torch.int32
if args.use_int32_inputs else torch.int64,
test_cases_per_run=args.test_cases,
total_runs=args.test_runs,
verbose=args.verbose,
)
latency = gpt2helper.test_performance(
session,
model,
device,
is_io_float16,
total_runs=100,
use_io_binding=True,
model_class=args.model_class,
has_position_ids=use_padding,
has_attention_mask=use_padding,
input_ids_dtype=torch.int32
if args.use_int32_inputs else torch.int64,
position_ids_dtype=torch.int32
if args.use_int32_inputs else torch.int64,
attention_mask_dtype=torch.int32
if args.use_int32_inputs else torch.int64,
batch_size=8,
sequence_length=1,
past_sequence_length=32,
)
if args.precision == Precision.FLOAT16:
logger.info(f"fp16 conversion parameters:{fp16_params}")
# Write results to file
import csv
from onnxruntime import __version__ as ort_version
latency_name = get_latency_name()
csv_file_existed = os.path.exists(csv_filename)
with open(csv_filename, mode="a", newline="") as csv_file:
column_names = [
"experiment",
"run_id",
"model_name",
"model_class",
"gpu",
"precision",
"optimizer",
"test_cases",
"runs",
"keep_io_types",
"io_block_list",
"op_block_list",
"node_block_list",
"force_fp16_initializers",
"auto_mixed_precision",
"ORT_TRANSFORMER_OPTIONS",
"ORT_CUDA_GEMM_OPTIONS",
"onnxruntime",
latency_name,
"top1_match_rate",
"onnx_size_in_MB",
"diff_50_percentile",
"diff_90_percentile",
"diff_95_percentile",
"diff_99_percentile",
"diff_pass_rate",
"nan_rate",
"top1_match_rate_per_run",
]
csv_writer = csv.DictWriter(csv_file, fieldnames=column_names)
if not csv_file_existed:
csv_writer.writeheader()
row = {
"experiment": experiment_name,
"run_id": run_id,
"model_name": args.model_name_or_path,
"model_class": args.model_class,
"gpu": args.use_gpu,
"precision": args.precision,
"optimizer": args.optimize_onnx,
"test_cases": args.test_cases,
"runs": args.test_runs,
"keep_io_types": args.keep_io_types,
"io_block_list": args.io_block_list,
"op_block_list": args.op_block_list,
"node_block_list": args.node_block_list,
"force_fp16_initializers": args.force_fp16_initializers,
"auto_mixed_precision": args.auto_mixed_precision,
"ORT_TRANSFORMER_OPTIONS":
os.getenv("ORT_TRANSFORMER_OPTIONS"),
"ORT_CUDA_GEMM_OPTIONS": os.getenv("ORT_CUDA_GEMM_OPTIONS"),
"onnxruntime": ort_version,
latency_name: f"{latency:.2f}",
"diff_50_percentile": parity_result["max_diff_percentile_50"],
"diff_90_percentile": parity_result["max_diff_percentile_90"],
"diff_95_percentile": parity_result["max_diff_percentile_95"],
"diff_99_percentile": parity_result["max_diff_percentile_99"],
"diff_pass_rate": parity_result["diff_pass_rate"],
"nan_rate": parity_result["nan_rate"],
"top1_match_rate": parity_result["top1_match_rate"],
"top1_match_rate_per_run":
parity_result["top1_match_rate_per_run"],
"onnx_size_in_MB": "{}".format(model_size_in_MB),
}
logger.info(f"result: {row}")
result.update(row)
csv_writer.writerow(row)
if args.input_test_file:
test_inputs = []
# Each line of test file is a JSON string like:
# {"input_ids": [[14698, 257, 1310, 13688, 319, 326]]}
with open(args.input_test_file) as read_f:
for _, line in enumerate(read_f):
line = line.rstrip()
data = json.loads(line)
input_ids = torch.from_numpy(
numpy.asarray(data["input_ids"],
dtype=numpy.int64)).to(device)
if use_padding:
if "attention_mask" in data:
numpy_float = numpy.float16 if is_io_float16 else numpy.float32
attention_mask = torch.from_numpy(
numpy.asarray(data["attention_mask"],
dtype=numpy_float)).to(device)
else:
padding = -1
attention_mask = (input_ids != padding).type(
torch.float16 if is_io_float16 else torch.float32)
input_ids.masked_fill_(input_ids == padding, 0)
if "position_ids" in data:
position_ids = torch.from_numpy(
numpy.asarray(data["position_ids"],
dtype=numpy.int64)).to(device)
else:
position_ids = attention_mask.long().cumsum(-1) - 1
position_ids.masked_fill_(position_ids < 0, 0)
inputs = {
"input_ids":
input_ids.to(torch.int32)
if args.use_int32_inputs else input_ids,
"position_ids":
position_ids.to(torch.int32)
if args.use_int32_inputs else position_ids,
"attention_mask":
attention_mask.to(torch.int32)
if args.use_int32_inputs else attention_mask,
}
else:
inputs = {
"input_ids":
input_ids.to(torch.int32)
if args.use_int32_inputs else input_ids
}
if model_type == "beam_search_step" or model_type == "configurable_one_step_search":
beam_select_idx = torch.zeros([1,
input_ids.shape[0]]).long()
input_log_probs = torch.zeros([input_ids.shape[0], 1])
input_unfinished_sents = torch.ones(
[input_ids.shape[0], 1], dtype=torch.bool)
inputs.update({
"beam_select_idx":
beam_select_idx,
"input_log_probs":
input_log_probs,
"input_unfinished_sents":
input_unfinished_sents,
})
test_inputs.append(inputs)
gpt2tester.test_generation(
session,
model,
device,
test_inputs,
precision=args.precision,
model_class=args.model_class,
top_k=20,
top_k_no_order=True,
max_steps=24,
max_inputs=0,
verbose=args.verbose,
save_test_data=3,
save_test_data_dir=Path(output_path).parent,
)
logger.info(f"Done. Output model: {output_path}")
return result
datePattern = re.compile(
r"""^(.*?)((0|1)?\d)-((0|1|2|3)?\d)-((19|20)?\d\d)(.*?)$""", re.VERBOSE)
#zeroPattern = re.compile(r"""^(.*?)(0?)(.*?)$""", re.VERBOSE)
# TODO: Loop over the files in the working directory.
for amerFile in os.listdir('/somedir'):
mo = datePattern.search(amerFile)
if mo == None:
continue
before = mo.group[1]
month = mo.group[2]
day = mo.group[4]
year = mo.group[6]
after = mo.group[8]
euroFile = before + day + '-' + month + '-' + year + after
absWorkingDir = os.path.abspath('.')
amerFile = os.path.join(absWorkingDir, amerFile)
euroFile = os.path.join(absWorkingDir, euroFile)
print('Renaming "%s" to "%s" ...' % (amerFile, euroFile))
shutil.move(amerFile, euroFile)
# TODO: Skip files without a date.
# TODO: Get the different parts of the filename.
# TODO: Form the European-style filename.
# TODO: Get the full, absolute file paths.
# TODO: Rename the files.
def main():
global connect, cursor
try:
noise_query=[]
file_list=sorted(glob.glob('*.txt')) # by default sort by name; key = lambda file: os.path.getctime(file)) # find all txt in the folders
# try:
we_have_data=False
for file_temp in file_list:
# print(file_temp)
with open(file_temp, 'r', encoding='cp1251') as txt:
entry = list()
all_strings=txt.readlines()
for string in all_strings:
value=string.strip().split('=')[1]
entry.append(value)
if len(entry)==0:
continue
# print('ENTRY', entry)
sql_query = single_parse(entry)
# CHECK, THAT WE HAVE time in air > than this noise time (means potential aircraft already in DB)
if (sql_query[1].startswith("VNK")):
SQL = '''
SELECT (time_track) FROM eco.tracks ORDER BY time_track desc LIMIT 1;
'''
elif (sql_query[1].startswith("OMSK")):
SQL = '''
SELECT (time_track) FROM omsk.tracks ORDER BY time_track desc LIMIT 1;
'''
cursor.execute(SQL)
last_time_air= cursor.fetchall()
# print(last_time_air1)
if len(last_time_air)>0: # the table can be empty..
last_time_air=last_time_air[0][0]
else:
shutil.move(file_temp, path+"/no_aircraft/"+file_temp)
continue # go to next file
# print('LAST AIR:', last_time_air)
# print('CURRENT NOISE:', sql_query[0])
if last_time_air-sql_query[0]>minimum_time:#minimum_time: # if noise_time is greater than noise datetime by 10 sec -> fine
# print('AIR IS ALREADY IN DB')
# SQL = '''
# SELECT (track, distance_1, time_track) FROM eco.tracks WHERE time_track - (%s) <= INTERVAL '10 seconds' and (%s)-time_track <= INTERVAL '10 seconds' and distance_1 IS NOT NULL ORDER BY distance_1 asc LIMIT 1;
# '''
# dont forget make index on time_track by CREATE INDEX idx_time_track ON eco.tracks ( time_track ); check index, via \d eco.tracks;
# CREATE INDEX time_dist_track ON eco.tracks (time_track, distance_1, track);
if (sql_query[1].startswith("VNK")):
SQL='''
SELECT (track, distance_1, time_track)
FROM eco.tracks
WHERE time_track >= (%s)::timestamp - INTERVAL '10 seconds'
and time_track <= (%s)::timestamp + INTERVAL '10 seconds'
and distance_1 IS NOT NULL
ORDER BY distance_1 asc LIMIT 1;
'''
elif (sql_query[1].startswith("OMSK")):
SQL='''
SELECT (track, distance_1, time_track)
FROM omsk.tracks
WHERE time_track >= (%s)::timestamp - INTERVAL '10 seconds'
and time_track <= (%s)::timestamp + INTERVAL '10 seconds'
and distance_1 IS NOT NULL
ORDER BY distance_1 asc LIMIT 1;
'''
# print('SQL QUERY', sql_query)
data = (sql_query[0],sql_query[0])
cursor.execute(SQL, data)
answer= cursor.fetchall()
if len(answer)==0: ##
shutil.move(file_temp, path+"/no_aircraft/"+file_temp)
track_distance=(None,None,None)
else:
we_have_data=True
track_distance=make_tuple(answer[0][0])
os.remove(file_temp)
if (sql_query[1].startswith("VNK")):
SQL = '''
INSERT INTO eco.noise (
time_noise, base_name, stat_1, stat_2, stat_3, leq, slow, spectrum, meteo_stat, temperature,
humadity, presure, wind, dir, gps_coordinate, gps_stat, temperature_core, temperature_mb, temperature_hdd, free_hdd,
ups_stat, ups_mode, ups_time, track, distance, aircraft_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING;
'''
elif (sql_query[1].startswith("OMSK")):
SQL = '''
INSERT INTO omsk.noise (
time_noise, base_name, stat_1, stat_2, stat_3, leq, slow, spectrum, meteo_stat, temperature,
humadity, presure, wind, dir, gps_coordinate, gps_stat, temperature_core, temperature_mb, temperature_hdd, free_hdd,
ups_stat, ups_mode, ups_time, track, distance, aircraft_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING;
'''
data = sql_query + track_distance
noise_query.append(cursor.mogrify(SQL, data).decode('utf-8'))
# do it after processing of the all txt list
if we_have_data:
full_noise_query=''.join([x for x in noise_query])
# print(full_noise_query)
cursor.execute(full_noise_query)
connect.commit()
except psycopg2.ProgrammingError as e:
connect.rollback()
print('something wrong with file at time' +str(e) + str(datetime.now().strftime('%Y-%m-%d-%H-%M-%S')))
shutil.move(txt_temp, path+"/wrong_files/")
# except psycopg2.InterfaceError as e:
# print('AZAZA')
except Exception as e:
print(str(e))
try:
connect = psycopg2.connect(database='eco_db', user='******', host='localhost', password='******', port=5432)
cursor = connect.cursor()
except Exception as e:
print(str(e))
def decompile_apk(apk_path, output_path, verbose):
print("[+] Decompiling the apk\n")
if verbose:
stdout = None
stderr = None
else:
stdout = DEVNULL
stderr = DEVNULL
if not os.path.exists(apk_path):
print("[-] Error: couldn't find the apk!")
return
apk_name = os.path.splitext(os.path.basename(apk_path))[0]
if os.path.exists("temp"):
print("[~] Removing old temp directory")
shutil.rmtree("temp")
print("[+] Creating temp directory")
os.makedirs("temp")
apk_zip = "temp/" + apk_name + ".zip"
shutil.copy2(apk_path, apk_zip)
apk_unziped_dir = "temp/" + apk_name + "_unziped"
os.makedirs(apk_unziped_dir)
zip_ref = zipfile.ZipFile(apk_zip, 'r')
zip_ref.extractall(apk_unziped_dir)
zip_ref.close()
apk_classes = apk_unziped_dir + "/classes.dex"
if not os.path.exists(apk_classes):
print("[-] Error: the apk doesn't have the classes.dex")
return
print("[+] Getting the jar")
apk_jar = "temp/" + apk_name + ".jar"
call(dex2jar_path + " " + apk_classes + " -o " + apk_jar,
stdout=stdout, stderr=stderr, shell=True)
print("[+] Decompiling the jar")
apk_java = "temp/" + apk_name + "_java/src"
call(jd_path + " " + apk_jar + " -od " + apk_java,
stdout=stdout, stderr=stderr, shell=True)
print("[+] Reverse engineering the apk")
apk_re = r"C:\Users\jake_\OneDrive\Desktop\Macquarie University\Personal Projects\Cybersecurity\Django\three\mysite\apkDownloads" + apk_name + "_re"
call(apktool_path + " d " + apk_path + " -o " + apk_re,
stdout=stdout, stderr=stderr, shell=True)
print("[+] Organizing everything")
output_dir = os.path.join(output_path, apk_name)
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
os.makedirs(output_dir)
print("[+] Moving reverse engineering files")
re_list = os.listdir(apk_re)
for re_files in re_list:
shutil.move(os.path.join(apk_re, re_files), output_dir)
print("[+] Moving java files")
shutil.move(apk_java, output_dir)
if os.path.exists("temp"):
print("[~] Removing temp directory")
shutil.rmtree("temp")
print("\n[+] Done decompiling the apk")
for path, subdirs, files in os.walk(videos_dir):
for name in files:
filenames.append(os.path.join(path, name))
size_dir = []
for file in filenames:
size_dir.append((file.split('/')[-1], file, os.path.getsize(file)))
#print((file.split('/')[-1], file, os.path.getsize(file)))
import operator
size_dir.sort(key=operator.itemgetter(2))
#print(size_dir)
lstoflst = []
for i in range(nparts):
lstoflst.append(size_dir[i::nparts])
print(len(lstoflst))
print(sum(len(lst) for lst in lstoflst))
for lst in lstoflst:
print(sum(pair[2] for pair in lst))
import shutil
for i in range(nparts):
newdir = '/home/wuzhenyu_sjtu/DAN_sbu/SBU/SBU_videos/clean_version/part{}'.format(
i)
if not os.path.exists(newdir):
os.makedirs(newdir)
for file in lstoflst[i]:
shutil.move(file[1], os.path.join(newdir, file[0]))
check = ModelCheckpoint(
filepath='./test/check/{epoch:02d}-{val_loss:.4f}.hdf5',
save_best_only=True,
save_weights_only=False)
model.fit([hite_x_train, samsung_x_train],
y_train,
batch_size=500,
epochs=500,
validation_split=0.4,
callbacks=[early, check])
## best값 폴더 밖으로 빼고 나머지 다 지우는 코드
tmp = os.path.dirname(os.path.realpath(__file__))
bestfile = os.listdir(tmp + '\\check')[-1]
shutil.move(tmp + '\\check' + '\\' + bestfile, tmp + '\\' + bestfile)
#os.rename(bestfile, 'Test0602_kkt.hdf5')
if os.path.isdir(tmp + '\\check'):
shutil.rmtree(tmp + '\\check')
os.mkdir(tmp + '\\check')
# best모델 불러오기
model = load_model(tmp + '\\' + bestfile) #Test0602_kkt.hdf5')
## 평가
from sklearn.metrics import r2_score
y_pred = model.predict([hite_x_test, samsung_x_test])
r2_y = r2_score(y_test, y_pred)
print("결정계수 : ", r2_y)
answer = model.predict([hite_x_predict, samsung_x_predict])
def ModifyFileLocation(root_directory, filename):
new_file_location = root_directory + '\\' + DetermineSubdirectory(filename) + '\\' + filename
shutil.move(root_directory + '\\' + filename, root_directory + '\\' + DetermineSubdirectory(filename) + '\\' + filename)
return new_file_location
import os
import shutil
import sys
import argparse
import uuid
import fnmatch
parser = argparse.ArgumentParser()
parser.add_argument("--dir", help="Source Path")
args = parser.parse_args()
tilefiles = []
for root, dirs, files in os.walk(args.dir):
for name in fnmatch.filter(files, "*title*"):
tilefiles.append(os.path.join(root, name))
if len(tilefiles) == 0:
sys.stdout("Error no title file!")
for titlefile_path in tilefiles:
unsortedfile = os.path.join(os.path.dirname(titlefile_path),
str(uuid.uuid1()) + ".pdf")
try:
shutil.move(titlefile_path, unsortedfile)
except:
print("can not move file %s -> %s" % (titlefile_path, unsortedfile))
def main():
try:
os.mkdir("results")
except:
pass
try:
os.mkdir("gfx")
except:
pass
sumoHome = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', '..', '..'))
if "SUMO_HOME" in os.environ:
sumoHome = os.environ["SUMO_HOME"]
sumo = os.environ.get("SUMO_BINARY", os.path.join(sumoHome, 'bin', 'sumo'))
assert (sumo)
for f1 in range(int(flow1def[0]), int(flow1def[1]), int(flow1def[2])):
pWE = float(f1) / 3600 # [veh/s]
pEW = pWE
for f2 in range(int(flow2def[0]), int(flow2def[1]), int(flow2def[2])):
pNS = float(f2) / 3600 # [veh/s]
pSN = pNS
print("Computing for %s<->%s" % (f1, f2))
buildDemand(simSteps, pWE, pEW, pNS, pSN)
for t in types:
print(" for tls-type %s" % t)
patchTLSType('input_additional_template.add.xml', '%tls_type%',
'input_additional.add.xml', t)
args = [
sumo,
'--no-step-log',
#'--no-duration-log',
#'--verbose',
#'--duration-log.statistics',
'--net-file',
'input_net.net.xml',
'--route-files',
'input_routes.rou.xml',
'--additional-files',
'input_additional.add.xml',
'--tripinfo-output',
'results/tripinfos_%s_%s_%s.xml' % (t, f1, f2),
'--summary-output',
'results/summary_%s_%s_%s.xml' % (t, f1, f2),
'--device.emissions.probability',
'1',
'--queue-output',
'results/queue_%s_%s_%s.xml' % (t, f1, f2),
]
retCode = subprocess.call(args)
shutil.move("results/e2_output.xml",
"results/e2_output_%s_%s_%s.xml" % (t, f1, f2))
shutil.move("results/e2_tl0_output.xml",
"results/e2_tl0_output_%s_%s_%s.xml" % (t, f1, f2))
shutil.move("results/edgeData_3600.xml",
"results/edgeData_3600_%s_%s_%s.xml" % (t, f1, f2))
shutil.move("results/laneData_3600.xml",
"results/laneData_3600_%s_%s_%s.xml" % (t, f1, f2))
shutil.move(
"results/edgesEmissions_3600.xml",
"results/edgesEmissions_3600_%s_%s_%s.xml" % (t, f1, f2))
shutil.move(
"results/lanesEmissions_3600.xml",
"results/lanesEmissions_3600_%s_%s_%s.xml" % (t, f1, f2))
shutil.move("results/TLSStates.xml",
"results/TLSStates_%s_%s_%s.xml" % (t, f1, f2))
shutil.move(
"results/TLSSwitchTimes.xml",
"results/TLSSwitchTimes_%s_%s_%s.xml" % (t, f1, f2))
shutil.move(
"results/TLSSwitchStates.xml",
"results/TLSSwitchStates_%s_%s_%s.xml" % (t, f1, f2))
def 模具_判断子目录元组目录列表个数(self):
print('==================')
for 根路径, 元组目录列表, 文件列表 in os.walk(
self.子目录): # "."目录,topdown=False先子文件夹再到元组文件夹
print('一层元组目录列表:', 元组目录列表)
if len(元组目录列表) != 0: # break # 结束循环 continue # 跳过当前循环,继续进行下一轮循环
元组目录列表与文件列表 = os.listdir(self.子目录) # 分离出目录列表与文件列表
for 子目录或文件 in 元组目录列表与文件列表:
元组目录的子目录或文件路径 = os.path.join(self.子目录,
子目录或文件) # 合并为完全 的 访问路径
if os.path.isdir(元组目录的子目录或文件路径): # 判断为 目录 文件夹
self.目录路径 = 元组目录的子目录或文件路径
self.模具_压缩()
self.模具_创建移动文件目录()
try:
移动文件名 = str(self.压缩目录名).replace(
"下载种子目录4", "压缩-下载种子") # 替换 , 1) 次数 1
if not os.path.exists(移动文件名): # 必有条件选择,否则出错
shutil.move(self.压缩目录名, self.移动文件目录名)
else: # 否则
os.unlink(移动文件名)
shutil.move(self.压缩目录名, self.移动文件目录名)
print('移动至目录:', self.移动文件目录名)
except (FileNotFoundError, OSError) as 异常:
print('异常# 跳过当前循环,继续进行', 异常)
continue # 跳过当前循环,继续进行下一轮循环
else: # 否则 为文件
文件路径 = 元组目录的子目录或文件路径
self.模具_创建移动文件目录()
移动文件名 = str(文件路径).replace("下载种子目录4",
"压缩-下载种子") # 替换 , 1) 次数 1
if not os.path.exists(移动文件名): # 必有条件选择,否则出错
try:
shutil.copy(文件路径, self.移动文件目录名)
print(文件路径, '复制文件:', self.移动文件目录名)
except (FileNotFoundError, OSError) as 异常:
print('异常# 跳过当前循环,继续进行', 异常)
continue # 跳过当前循环,继续进行下一轮循环
else: # 否则
print('复制文件已存在:跳过当前文件')
continue # 跳过当前循环,继续进行下一轮循环
else: # 否则
if len(文件列表) != 0:
self.目录路径 = self.子目录
self.模具_压缩()
移动文件名 = str(self.压缩目录名).replace(
"下载种子目录4\动漫", "压缩-下载种子\动漫打包") # 替换 , 1) 次数 1
try:
if not os.path.exists(移动文件名): # 必有条件选择,否则出错
shutil.move(self.压缩目录名, self.移动文件目录名)
else: # 否则
os.unlink(移动文件名) #删除原来 文件
shutil.move(self.压缩目录名, self.移动文件目录名)
except (FileNotFoundError, OSError) as 异常:
print('异常# 跳过当前循环,继续进行', 异常)
continue # 跳过当前循环,继续进行下一轮循环
print('移动至目录:', 移动文件名)
break # 结束循环
found = extractext(srcfile)
if found:
size = size + (os.path.getsize(srcfile) / (1024 * 1024.0))
named = os.path.join(new_folder, folder)
try:
#Create folder
os.mkdir(named)
log(0, "Successfully created the directory %s " % named)
except OSError:
log(1, "Creation of the directory %s failed" % named)
destfile = os.path.join(named, filename)
destfile
if not os.path.exists(destfile):
shutil.move(srcfile, destfile)
log(0, "Archived '" + filename + "'.")
files_success = files_success + 1
else:
shutil.move(srcfile, errorfile)
log(1, "File Exists '" + filename + "'.")
files_with_errors = files_with_errors + 1
else:
shutil.move(srcfile, errorfile)
log(1, "Archived '" + filename + "'.")
files_with_errors = files_with_errors + 1
log(
0, "Successfully achieved " + str(files_success) + " files, totalling " +
str(round(size, 2)) + "MB. files with errors " + str(files_with_errors))
end(0)
def _record(args, dom, tmpdir):
def create_ffmpeg_stream(display):
path = os.path.join(tmpdir, '{:03}-{}x{}.mp4'.format(
display.index, display.width, display.height))
return FFmpegRawStream(
path = path,
display = display,
framerate = args.framerate,
outcodec = args.vcodec,
loglevel = logging_to_ffmpeg_loglevel(args.loglevel),
)
domain_wait(dom, libvirt.VIR_DOMAIN_RUNNING)
with TtyCbreakMode():
# Open spice session
sp = SpiceRecorder(dom,
framerate = args.framerate,
create_display_stream = create_ffmpeg_stream,
)
sp.open()
# Record raw video
def periodic_update(sp):
qprint("\r" + " "*80, end="")
qprint("\r{:<20}{:<20}{:<20}".format(
"{:0.02f} sec".format(sp.elapsed_time),
"{} frames".format(sp.frames_recorded),
format_datasize(sp.bytes_recorded),
), end="")
def on_stopped(sp, msg):
qprint("\nRecording stopped:", msg)
sp.connect("periodic-update", periodic_update)
sp.connect("recording-stopped", on_stopped)
qprint("Recording... Press Q to stop")
sp.run()
# Finalize all intermediate FFmpeg processes
for d in sp.displays:
rc = d.outfile.close()
qprint("-"*80)
qprint("Recorded displays:")
maxw, maxh = 0, 0
for n,d in enumerate(sp.displays):
qprint(" {}: {}x{} {:>4} frames {:>10} {:0.02f} sec".format(n, d.width, d.height,
d.frames_recorded,
format_datasize(os.path.getsize(d.outfile.name)),
d.duration))
maxw = max(maxw, d.width)
maxh = max(maxh, d.height)
qprint("Final: {}x{}".format(maxw, maxh))
qprint("-"*80)
# Filter out displays with no frames
displays = [d for d in sp.displays if d.frames_recorded]
if len(displays) == 1 and H264_PIX_FMT_INTERMEDIATE(displays[0]) == H264_PIX_FMT_FINAL:
# Optimization: use the only intermediate video as the final
d = displays[0]
src = d.outfile.name
d.outfile = None
logging.info("Moving {} to {}".format(src, args.output))
shutil.move(src, args.output)
else:
# Convert video
qprint("\nDone recording. Converting...")
convert_concat_videos(
displays = displays,
framerate = args.framerate,
outcodec = args.vcodec,
outpath = args.output,
loglevel = logging_to_ffmpeg_loglevel(args.loglevel),
)
qprint("\n{} written!".format(args.output))
import os
import sys
import shutil
input = sys.argv[1]
filecount = 1
#列出輸入參數位置底下的文件
os.mkdir("YOLO")
for filename in os.listdir(input):
os.system(
"E:\darknet\\build\darknet\\x64\darknet.exe detect cfg\yolov3.cfg weights\yolov3.weights "
+ input + "\\" + filename)
os.rename("predictions.jpg", str(filecount) + ".jpg")
shutil.move(str(filecount) + ".jpg", "YOLO")
filecount += 1
def download_and_prepare(
self,
download_config: Optional[DownloadConfig] = None,
download_mode: Optional[GenerateMode] = None,
ignore_verifications: bool = False,
save_infos: bool = False,
try_from_hf_gcs: bool = True,
dl_manager: Optional[DownloadManager] = None,
**download_and_prepare_kwargs,
):
"""Downloads and prepares dataset for reading.
Args:
download_config (Optional ``nlp.DownloadConfig``: specific download configuration parameters.
download_mode (Optional `nlp.GenerateMode`): select the download/generate mode - Default to REUSE_DATASET_IF_EXISTS
ignore_verifications (bool): Ignore the verifications of the downloaded/processed dataset information (checksums/size/splits/...)
save_infos (bool): Save the dataset information (checksums/size/splits/...)
try_from_hf_gcs (bool): If True, it will try to download the already prepared dataset from the Hf google cloud storage
dl_manager (Optional ``nlp.DownloadManager``): specific Download Manger to use
"""
download_mode = GenerateMode(download_mode or GenerateMode.REUSE_DATASET_IF_EXISTS)
data_exists = os.path.exists(self._cache_dir)
if data_exists and download_mode == REUSE_DATASET_IF_EXISTS:
logger.info("Reusing dataset %s (%s)", self.name, self._cache_dir)
return
# Currently it's not possible to overwrite the data because it would
# conflict with versioning: If the last version has already been generated,
# it will always be reloaded and cache_dir will be set at construction.
if data_exists and download_mode != REUSE_CACHE_IF_EXISTS:
raise ValueError(
"Trying to overwrite an existing dataset {} at {}. A dataset with "
"the same version {} already exists. If the dataset has changed, "
"please update the version number.".format(self.name, self._cache_dir, self.config.version)
)
logger.info("Generating dataset %s (%s)", self.name, self._cache_dir)
if not is_remote_url(self._cache_dir): # if cache dir is local, check for available space
os.makedirs(self._cache_dir_root, exist_ok=True)
if not utils.has_sufficient_disk_space(self.info.size_in_bytes or 0, directory=self._cache_dir_root):
raise IOError(
"Not enough disk space. Needed: {} (download: {}, generated: {})".format(
utils.size_str(self.info.size_in_bytes or 0),
utils.size_str(self.info.download_size or 0),
utils.size_str(self.info.dataset_size or 0),
)
)
@contextlib.contextmanager
def incomplete_dir(dirname):
"""Create temporary dir for dirname and rename on exit."""
if is_remote_url(dirname):
yield dirname
else:
tmp_dir = dirname + ".incomplete"
os.makedirs(tmp_dir)
try:
yield tmp_dir
if os.path.isdir(dirname):
shutil.rmtree(dirname)
os.rename(tmp_dir, dirname)
finally:
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
# Try to download the already prepared dataset files
if try_from_hf_gcs:
try:
# Create a tmp dir and rename to self._cache_dir on successful exit.
with incomplete_dir(self._cache_dir) as tmp_data_dir:
# Temporarily assign _cache_dir to tmp_data_dir to avoid having to forward
# it to every sub function.
with utils.temporary_assignment(self, "_cache_dir", tmp_data_dir):
relative_data_dir = self._relative_data_dir(with_version=True)
reader = ArrowReader(self._cache_dir, self.info)
# use reader instructions to download the right files
reader.download_from_hf_gcs(self._cache_dir, relative_data_dir)
downloaded_info = DatasetInfo.from_directory(self._cache_dir)
self.info.update(downloaded_info)
# download post processing resources
remote_cache_dir = os.path.join(HF_GCP_BASE_URL, relative_data_dir)
for resource_file_name in self._post_processing_resources().values():
if "/" in resource_file_name:
raise ValueError(
"Resources shouldn't be in a sub-directory: {}".format(resource_file_name)
)
try:
resource_path = utils.cached_path(os.path.join(remote_cache_dir, resource_file_name))
shutil.move(resource_path, os.path.join(self._cache_dir, resource_file_name))
except ConnectionError:
logger.info(
"Couldn't download resourse file {} from Hf google storage.".format(
resource_file_name
)
)
logger.info("Dataset downloaded from Hf google storage.")
print(
f"Dataset {self.name} downloaded and prepared to {self._cache_dir}. "
f"Subsequent calls will reuse this data."
)
return
except (DatasetNotOnHfGcs, MissingFilesOnHfGcs):
logger.info("Dataset not on Hf google storage. Downloading and preparing it from source")
# Print is intentional: we want this to always go to stdout so user has
# information needed to cancel download/preparation if needed.
# This comes right before the progress bar.
print(
f"Downloading and preparing dataset {self.info.builder_name}/{self.info.config_name} "
f"(download: {utils.size_str(self.info.download_size)}, generated: {utils.size_str(self.info.dataset_size)}, "
f"total: {utils.size_str(self.info.size_in_bytes)}) to {self._cache_dir}..."
)
if dl_manager is None:
if download_config is None:
download_config = DownloadConfig()
download_config.cache_dir = os.path.join(self._cache_dir_root, "downloads")
download_config.force_download = download_mode == FORCE_REDOWNLOAD
dl_manager = DownloadManager(
dataset_name=self.name, download_config=download_config, data_dir=self.config.data_dir
)
if self.manual_download_instructions is not None:
assert (
dl_manager.manual_dir is not None
), "The dataset {} with config {} requires manual data. \n Please follow the manual download instructions: {}. \n Manual data can be loaded with `nlp.load_dataset({}, data_dir='<path/to/manual/data>')".format(
self.name, self.config.name, self.manual_download_instructions, self.name
)
# Create a tmp dir and rename to self._cache_dir on successful exit.
with incomplete_dir(self._cache_dir) as tmp_data_dir:
# Temporarily assign _cache_dir to tmp_data_dir to avoid having to forward
# it to every sub function.
with utils.temporary_assignment(self, "_cache_dir", tmp_data_dir):
verify_infos = not save_infos and not ignore_verifications
self._download_and_prepare(
dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
)
# Sync info
self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())
self.info.download_checksums = dl_manager.get_recorded_sizes_checksums()
self.info.size_in_bytes = self.info.dataset_size + self.info.download_size
# Save info
self._save_info()
# Save to datasetinfos
if save_infos:
DatasetInfosDict(**{self.config.name: self.info}).write_to_directory(self.get_imported_module_dir())
print(
f"Dataset {self.name} downloaded and prepared to {self._cache_dir}. "
f"Subsequent calls will reuse this data."
)
# ORGANISING DESKTOP
import os
import shutil
print('Please wait, process is running...')
systemdrivename=os.getenv('systemdrive')
username=os.getenv('username')
homedir=systemdrivename+'\\users\\'+username
for (dirname,dirs,files) in os.walk(homedir+'\\desktop'):
for file in files:
try:
filepath=os.path.join(dirname,file)
ext=file.split('.')[-1]
if ext!='exe' and ext!='lnk':
if os.path.isdir(homedir+'\\documents\\'+ext):
pass
else: os.mkdir(homedir+'\\documents\\'+ext)
if os.path.exists(homedir+'\\documents\\'+ext+'\\'+file):
os.remove(homedir+'\\documents\\'+ext+'\\'+file)
shutil.move(filepath,homedir+'\\documents\\'+ext)
except:
pass
break
input('Desktop has been organized.')
ax3.plot(time_total, viscosities[-1], "x")
ax3.autoscale()
fig3.canvas.draw()
ax4.plot(time_total, energy, "x")
ax4.autoscale()
fig4.canvas.draw()
ax5.plot(time_total, factor, 'x')
ax5.autoscale()
fig5.canvas.draw()
plt.pause(1e-4)
if mpi.rank == 0:
if len(paths_sim) == 0:
to_print = ("gamma,nx,nu8 \n")
to_print += ("{},{},{} \n".format(
gamma, sim.params.oper.nx, params.nu_8))
mode_write = "w"
else:
to_print = ("{},{},{} \n".format(
gamma, sim.params.oper.nx, params.nu_8))
mode_write = "a"
write_to_file(path_file_write, to_print, mode=mode_write)
shutil.move(sim.params.path_run, path)
def state_cmd(self, item):
if item['cmd'] == 'PROCESS':
output_filename = self.pictures.get_next()
offset =[]
photos = []
if self.cfg.get("image__format") == "fancy":
#Create Image With Template
if self.cfg.get("image__background"):
out_img = cv.imread(self.cfg.get("image__background"))
else:
out_img = np.ones([self.image_size[1],self.image_size[0],3], dtype=np.uint8)*255
dpi = float(self.image_size[0])/6
dpm = int(dpi/25.4)
lr_off = dpm*20
new_w = int( (self.image_size[0] - (2*lr_off)) / 2 )
new_h = int( float(new_w)/1.5 )
thumb_size = ( new_w, new_h )
v_space = (self.image_size[1] - (2*new_h))
top_off = int(0.33*v_space)
mid_off = int(0.16*v_space)
bot_off = int(0.5 *v_space)
m = int(dpm)
offset.append( ( lr_off - m ,top_off ))
offset.append( ( lr_off + new_w +m, top_off ))
offset.append( ( lr_off - m , top_off + new_h + mid_off ) )
offset.append( ( lr_off + new_w + m , top_off + new_h + mid_off ) )
Logger.info(__name__, "fancy print thumb_size: {0}".format(thumb_size))
Logger.info(__name__, "fancy print thumb coords: {0}".format(offset))
else:
"""Assembles four pictures into a 2x2 grid
It assumes, all original pictures have the same aspect ratio as
the resulting image.
For the thumbnail sizes we have:
h = (H - 2 * a - 2 * b) / 2
w = (W - 2 * a - 2 * b) / 2
W
|---------------------------------------|
--- +---+-------------+---+-------------+---+ ---
| | | | a
| | +-------------+ +-------------+ | ---
| | | | | | | |
| | | 0 | | 1 | | | h
| | | | | | | |
| | +-------------+ +-------------+ | ---
H | | | | 2*b
| | +-------------+ +-------------+ | ---
| | | | | | | |
| | | 2 | | 3 | | | h
| | | | | | | |
| | +-------------+ +-------------+ | ---
| | | | a
--- +---+-------------+---+-------------+---+ ---
|---|-------------|---|-------------|---|
a w 2*b w a
"""
# Thumbnail size of pictures
outer_border = 100
inner_border = 20
thumb_box = ( int( self.image_size[0] / 2 ) ,
int( self.image_size[1] / 2 ) )
thumb_size = ( thumb_box[0] - outer_border - inner_border ,
thumb_box[1] - outer_border - inner_border )
offset.append( ( thumb_box[0] - inner_border - img.size[0] ,
thumb_box[1] - inner_border - img.size[1] ))
offset.append( ( thumb_box[0] + inner_border,
thumb_box[1] - inner_border - img.size[1] ))
offset.append( ( thumb_box[0] - inner_border - img.size[0] ,
thumb_box[1] + inner_border ) )
offset.append( ( thumb_box[0] + inner_border ,
thumb_box[1] + inner_border ) )
for i in range(4):
o = offset[i]
img = cv.imread(self.photo_set[i])
lar_thumb = cv.resize(img, self.image_size, interpolation=cv.INTER_AREA )
cv.imwrite( self.photo_set[i], lar_thumb, [int(cv.IMWRITE_JPEG_QUALITY), 82] )
sma_thumb = cv.resize(lar_thumb, thumb_size, interpolation=cv.INTER_AREA )
out_img[o[1]:o[1]+sma_thumb.shape[0], o[0]:o[0]+sma_thumb.shape[1]] = sma_thumb
newname = output_filename.replace(".","."+str(i)+".")
shutil.move( self.photo_set[i], newname )
photos.append(newname)
# Save assembled image
cv.imwrite( output_filename, out_img, [int(cv.IMWRITE_JPEG_QUALITY), 90] )
thumb_filename = output_filename.replace("/pic","/thumb.pic")
thumb = cv.resize(out_img, ( int(self.image_size[0]/6), int(self.image_size[1]/6)),
interpolation=cv.INTER_AREA )
cv.imwrite( thumb_filename, thumb, [int(cv.IMWRITE_JPEG_QUALITY), 90] )
self.final_photos= {'primary': output_filename,
'primary_thumb': thumb_filename,
'secondary' : photos }
self.ani_q_cmd_push("PROCESSPREVIEW")
elif item['cmd'] == 'PROCESSPREVIEW':
self.gpio.set('green_led', 1)
self.gpio.set('red_led', 1)
self.gameDisplay.fill((200,200,200))
img = pygame.image.load(self.final_photos['primary'])
ratio = 0.275
#ratio = 0.55
shrink = ( int(img.get_size()[0]*ratio), int(img.get_size()[1]*ratio))
img = pygame.transform.scale(img, shrink)
img_pos = ((self.disp_w-img.get_size()[0])/2+180,65)
if self.cfg.get("upload__enabled"):
pass
elif self.cfg.get("printer__enabled"):
self.gen_processing_menu("Print") #Regen with Print
else:
self.gen_processing_menu("Finish") #Regen with finish
self.ani_q_img_push( self.processing_menu, self.processing_menu_pos, 0.1, False, True)
self.ani_q_img_push( img, img_pos, 0.3, True, False)
self.ani_q_cmd_push("COMPLETE")
import os
import os.path
import shutil
# You can add more file formats here
image_formats = ["jpg", "png", "jpeg", "gif", "webp", "tiff"]
audio_formats = ["mp3", "wav"]
video_formats = ["mp4", "avi", "webm"]
docs_formats = ["ai", "ait", "txt", "rtf", "pdf"]
files = os.listdir("./")
for file in files:
if os.path.isfile(file) and file != "smartDirectoryManagementSystem.py":
ext = (file.split(".")[-1]).lower()
if ext in image_formats:
shutil.move(file, "images/" + file)
elif ext in audio_formats:
shutil.move(file, "audio/" + file)
elif ext in video_formats:
shutil.move(file, "videos/" + file)
elif ext in docs_formats:
shutil.move(file, "docs/" + file)
else:
shutil.move(file, "others/" + file)
def _restore_fstab(self, fstab):
"""Restore the saved fstab in rootfs"""
if fstab is None:
return
shutil.move(fstab + ".orig", fstab)
def Rfoil(coord_foil_name,foil_name,alpha,Re,Mach,Ncrit,obj):
#
## for function test
##alpha=7
##Re=1e6
##Mach=0
##Ncrit=9
##savepath='geo'
os.chdir('geo')
with open('rfoil.inp','w') as f:
f.write('load \n')
# f.write('airfoil.dat \n')
f.write('%s\n'%coord_foil_name)
f.write('airfoil\n')
f.write('ppar\n')
f.write('N \n')
f.write('300 \n\n')
f.write('isav \n')
f.write('isav.dat \n\n\n')
f.write('oper\n')
f.write('visc\n')
f.write('%e\n'%Re)
f.write('Mach\n')
f.write('%e\n'%Mach)
if Ncrit == 0:
f.write('vpar\n')
f.write('XTR\n')
f.write('0.05 \n')
f.write('0.05 \n\n')
else:
f.write('vpar\n')
f.write('N\n')
f.write('%g\n\n'%Ncrit)
f.write('pacc\n')
f.write('polar.dat \n')
f.write('\n')
f.write('arbl\n')
f.write('aseq\n')
f.write('%g\n'%np.min(alpha))
f.write('%g\n'%np.max(alpha))
f.write('1 \n')
#tf = strcmp(version,'RFOIL.v4.0');
#if tf == 1
# fprintf(fid,'y \n');
#end
f.write('svbl\n');
f.write('1 \n');
f.write('bl \n');
f.write('pacc\n');
f.write('\n');
f.write('\nquit\n');
f.close();
os.system("rfoil.exe < rfoil.inp")
# # # CALCULATE THE cl/cd
alpha_out=np.zeros(np.size(alpha), dtype=np.float)
cl=np.zeros(np.size(alpha), dtype=np.float)
cd=np.zeros(np.size(alpha), dtype=np.float)
polar_file = open('polar.dat', "r")
data = [row for row in polar_file]
for j in range(0,len(data)-13): # # # j is the number of AOAs
str_list = list(filter(None, data[j+13].split(' ')))
alpha_out[j]=float(str_list[0])
cl[j]=float(str_list[1])
cd[j]=float(str_list[2])
polar_file.close()
clOcd=cl/cd
if np.count_nonzero(alpha_out)==0 :
f2=(np.count_nonzero(alpha_out)+1.0)*1.0/len(alpha)
else:
f2=(np.count_nonzero(alpha_out))*1.0/len(alpha)
if obj==0: # # max cl
if len(cl)>1:
area=(cl[0:-1]+cl[1:])*(alpha_out[1:]-alpha_out[0:-1])/2 # # # for target max(cl)
area = area[~np.isnan(area)] # # # remove nan
target=np.array([np.sum(area)])
else:
target=np.array([cl])
else:
if len(cl)>1:
area=(clOcd[0:-1]+clOcd[1:])*(alpha_out[1:]-alpha_out[0:-1])/2 # # # for target max(cl/cd)
area = area[~np.isnan(area)] # # # remove nan
target=np.array([np.sum(area)])
else:
target=np.array([clOcd])
# # # save the aerodynamic data into sub_folder
files=os.listdir('.')
if not os.path.exists(foil_name):
os.makedirs(foil_name)
shutil.move(coord_foil_name,foil_name)
shutil.move('rfoil.inp',foil_name)
shutil.move('polar.dat',foil_name)
shutil.move('isav.dat',foil_name)
shutil.move('bl.cfx',foil_name)
shutil.move('bl.cpx',foil_name)
shutil.move('bl.dst',foil_name)
shutil.move('bl.tet',foil_name)
#for f in files:
# if not f.endswith("exe") and os.path.isfile(os.path.join(os.getcwd()+f)):
# shutil.move(f,foil_name)
os.chdir("..")
return {'lift':cl, 'drag':cd,'OBJ':target,'conver_alpha':f2}
def init_seafile_server():
version_stamp_file = get_version_stamp_file()
if exists(join(shared_seafiledir, 'seafile-data')):
if not exists(version_stamp_file):
update_version_stamp(os.environ['SEAFILE_VERSION'])
# sysbol link unlink after docker finish.
latest_version_dir='/opt/seafile/seafile-server-latest'
current_version_dir='/opt/seafile/' + get_conf('SEAFILE_SERVER', 'seafile-server') + '-' + read_version_stamp()
if not exists(latest_version_dir):
call('ln -sf ' + current_version_dir + ' ' + latest_version_dir)
loginfo('Skip running setup-seafile-mysql.py because there is existing seafile-data folder.')
return
loginfo('Now running setup-seafile-mysql.py in auto mode.')
env = {
'SERVER_NAME': 'seafile',
'SERVER_IP': get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com'),
'MYSQL_USER': '******',
'MYSQL_USER_PASSWD': str(uuid.uuid4()),
'MYSQL_USER_HOST': '%.%.%.%',
'MYSQL_HOST': get_conf('DB_HOST','127.0.0.1'),
# Default MariaDB root user has empty password and can only connect from localhost.
'MYSQL_ROOT_PASSWD': get_conf('DB_ROOT_PASSWD', ''),
}
# Change the script to allow mysql root password to be empty
# call('''sed -i -e 's/if not mysql_root_passwd/if not mysql_root_passwd and "MYSQL_ROOT_PASSWD" not in os.environ/g' {}'''
# .format(get_script('setup-seafile-mysql.py')))
# Change the script to disable check MYSQL_USER_HOST
call('''sed -i -e '/def validate_mysql_user_host(self, host)/a \ \ \ \ \ \ \ \ return host' {}'''
.format(get_script('setup-seafile-mysql.py')))
call('''sed -i -e '/def validate_mysql_host(self, host)/a \ \ \ \ \ \ \ \ return host' {}'''
.format(get_script('setup-seafile-mysql.py')))
setup_script = get_script('setup-seafile-mysql.sh')
call('{} auto -n seafile'.format(setup_script), env=env)
domain = get_conf('SEAFILE_SERVER_HOSTNAME', 'seafile.example.com')
proto = 'https' if is_https() else 'http'
with open(join(topdir, 'conf', 'seahub_settings.py'), 'a+') as fp:
fp.write('\n')
fp.write("""CACHES = {
'default': {
'BACKEND': 'django_pylibmc.memcached.PyLibMCCache',
'LOCATION': 'memcached:11211',
},
'locmem': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
},
}
COMPRESS_CACHE_BACKEND = 'locmem'""")
fp.write('\n')
fp.write("TIME_ZONE = '{time_zone}'".format(time_zone=os.getenv('TIME_ZONE',default='Etc/UTC')))
fp.write('\n')
fp.write('FILE_SERVER_ROOT = "{proto}://{domain}/seafhttp"'.format(proto=proto, domain=domain))
fp.write('\n')
# By default ccnet-server binds to the unix socket file
# "/opt/seafile/ccnet/ccnet.sock", but /opt/seafile/ccnet/ is a mounted
# volume from the docker host, and on windows and some linux environment
# it's not possible to create unix sockets in an external-mounted
# directories. So we change the unix socket file path to
# "/opt/seafile/ccnet.sock" to avoid this problem.
with open(join(topdir, 'conf', 'ccnet.conf'), 'a+') as fp:
fp.write('\n')
fp.write('[Client]\n')
fp.write('UNIX_SOCKET = /opt/seafile/ccnet.sock\n')
fp.write('\n')
# Disabled the Elasticsearch process on Seafile-container
# Connection to the Elasticsearch-container
if os.path.exists(join(topdir, 'conf', 'seafevents.conf')):
with open(join(topdir, 'conf', 'seafevents.conf'), 'r') as fp:
fp_lines = fp.readlines()
if '[INDEX FILES]\n' in fp_lines:
insert_index = fp_lines.index('[INDEX FILES]\n') + 1
insert_lines = ['es_port = 9200\n', 'es_host = elasticsearch\n', 'external_es_server = true\n']
for line in insert_lines:
fp_lines.insert(insert_index, line)
with open(join(topdir, 'conf', 'seafevents.conf'), 'w') as fp:
fp.writelines(fp_lines)
# After the setup script creates all the files inside the
# container, we need to move them to the shared volume
#
# e.g move "/opt/seafile/seafile-data" to "/shared/seafile/seafile-data"
files_to_copy = ['conf', 'ccnet', 'seafile-data', 'seahub-data', 'pro-data']
for fn in files_to_copy:
src = join(topdir, fn)
dst = join(shared_seafiledir, fn)
if not exists(dst) and exists(src):
shutil.move(src, shared_seafiledir)
call('ln -sf ' + join(shared_seafiledir, fn) + ' ' + src)
loginfo('Updating version stamp')
update_version_stamp(os.environ['SEAFILE_VERSION'])
def handle_entry(self, task, config, entry, siblings):
src = entry['location']
src_isdir = os.path.isdir(src)
src_path, src_name = os.path.split(src)
# get the proper path and name in order of: entry, config, above split
dst_path = entry.get(self.destination_field, config.get('to', src_path))
if config.get('rename'):
dst_name = config['rename']
elif entry.get('filename') and entry['filename'] != src_name:
# entry specifies different filename than what was split from the path
# since some inputs fill in filename it must be different in order to be used
dst_name = entry['filename']
else:
dst_name = src_name
try:
dst_path = entry.render(dst_path)
except RenderError as err:
raise plugin.PluginError('Path value replacement `%s` failed: %s' % (dst_path, err.args[0]))
try:
dst_name = entry.render(dst_name)
except RenderError as err:
raise plugin.PluginError('Filename value replacement `%s` failed: %s' % (dst_name, err.args[0]))
# Clean invalid characters with pathscrub plugin
dst_path = pathscrub(os.path.expanduser(dst_path))
dst_name = pathscrub(dst_name, filename=True)
# Join path and filename
dst = os.path.join(dst_path, dst_name)
if dst == entry['location']:
raise plugin.PluginWarning('source and destination are the same.')
if not os.path.exists(dst_path):
if task.options.test:
self.log.info('Would create `%s`', dst_path)
else:
self.log.info('Creating destination directory `%s`', dst_path)
os.makedirs(dst_path)
if not os.path.isdir(dst_path) and not task.options.test:
raise plugin.PluginWarning('destination `%s` is not a directory.' % dst_path)
# unpack_safety
if config.get('unpack_safety', entry.get('unpack_safety', True)):
count = 0
while True:
if count > 60 * 30:
raise plugin.PluginWarning('The task has been waiting unpacking for 30 minutes')
size = os.path.getsize(src)
time.sleep(1)
new_size = os.path.getsize(src)
if size != new_size:
if not count % 10:
self.log.verbose('File `%s` is possibly being unpacked, waiting ...', src_name)
else:
break
count += 1
src_file, src_ext = os.path.splitext(src)
dst_file, dst_ext = os.path.splitext(dst)
# Check dst contains src_ext
if config.get('keep_extension', entry.get('keep_extension', True)):
if not src_isdir and dst_ext != src_ext:
self.log.verbose('Adding extension `%s` to dst `%s`', src_ext, dst)
dst += src_ext
dst_file += dst_ext # this is used for sibling files. dst_ext turns out not to be an extension!
funct_name = 'move' if self.move else 'copy'
funct_done = 'moved' if self.move else 'copied'
if task.options.test:
self.log.info('Would %s `%s` to `%s`', funct_name, src, dst)
for s, ext in siblings.items():
# we cannot rely on splitext for extensions here (subtitles may have the language code)
d = dst_file + ext
self.log.info('Would also %s `%s` to `%s`', funct_name, s, d)
else:
# IO errors will have the entry mark failed in the base class
if self.move:
shutil.move(src, dst)
elif src_isdir:
shutil.copytree(src, dst)
else:
shutil.copy(src, dst)
self.log.info('`%s` has been %s to `%s`', src, funct_done, dst)
# further errors will not have any effect (the entry has been successfully moved or copied out)
for s, ext in siblings.items():
# we cannot rely on splitext for extensions here (subtitles may have the language code)
d = dst_file + ext
try:
if self.move:
shutil.move(s, d)
else:
shutil.copy(s, d)
self.log.info('`%s` has been %s to `%s` as well.', s, funct_done, d)
except Exception as err:
self.log.warning(str(err))
entry['old_location'] = entry['location']
entry['location'] = dst
if self.move and not src_isdir:
self.clean_source(task, config, entry)
def install_from_urls(self, urls):
if not self.env.user.has_group('base.group_system'):
raise AccessDenied()
# One-click install is opt-in - cfr Issue #15225
ad_dir = tools.config.addons_data_dir
if not os.access(ad_dir, os.W_OK):
msg = (_("Automatic install of downloaded Apps is currently disabled.") + "\n\n" +
_("To enable it, make sure this directory exists and is writable on the server:") +
"\n%s" % ad_dir)
_logger.warning(msg)
raise UserError(msg)
apps_server = urls.url_parse(self.get_apps_server())
OPENERP = odoo.release.product_name.lower()
tmp = tempfile.mkdtemp()
_logger.debug('Install from url: %r', urls)
try:
# 1. Download & unzip missing modules
for module_name, url in pycompat.items(urls):
if not url:
continue # nothing to download, local version is already the last one
up = urls.url_parse(url)
if up.scheme != apps_server.scheme or up.netloc != apps_server.netloc:
raise AccessDenied()
try:
_logger.info('Downloading module `%s` from OpenERP Apps', module_name)
response = requests.get(url)
response.raise_for_status()
content = response.content
except Exception:
_logger.exception('Failed to fetch module %s', module_name)
raise UserError(_('The `%s` module appears to be unavailable at the moment, please try again later.') % module_name)
else:
zipfile.ZipFile(io.BytesIO(content)).extractall(tmp)
assert os.path.isdir(os.path.join(tmp, module_name))
# 2a. Copy/Replace module source in addons path
for module_name, url in pycompat.items(urls):
if module_name == OPENERP or not url:
continue # OPENERP is special case, handled below, and no URL means local module
module_path = modules.get_module_path(module_name, downloaded=True, display_warning=False)
bck = backup(module_path, False)
_logger.info('Copy downloaded module `%s` to `%s`', module_name, module_path)
shutil.move(os.path.join(tmp, module_name), module_path)
if bck:
shutil.rmtree(bck)
# 2b. Copy/Replace server+base module source if downloaded
if urls.get(OPENERP):
# special case. it contains the server and the base module.
# extract path is not the same
base_path = os.path.dirname(modules.get_module_path('base'))
# copy all modules in the SERVER/odoo/addons directory to the new "odoo" module (except base itself)
for d in os.listdir(base_path):
if d != 'base' and os.path.isdir(os.path.join(base_path, d)):
destdir = os.path.join(tmp, OPENERP, 'addons', d) # XXX 'odoo' subdirectory ?
shutil.copytree(os.path.join(base_path, d), destdir)
# then replace the server by the new "base" module
server_dir = tools.config['root_path'] # XXX or dirname()
bck = backup(server_dir)
_logger.info('Copy downloaded module `odoo` to `%s`', server_dir)
shutil.move(os.path.join(tmp, OPENERP), server_dir)
#if bck:
# shutil.rmtree(bck)
self.update_list()
with_urls = [module_name for module_name, url in pycompat.items(urls) if url]
downloaded = self.search([('name', 'in', with_urls)])
installed = self.search([('id', 'in', downloaded.ids), ('state', '=', 'installed')])
to_install = self.search([('name', 'in', list(urls)), ('state', '=', 'uninstalled')])
post_install_action = to_install.button_immediate_install()
if installed or to_install:
# in this case, force server restart to reload python code...
self._cr.commit()
odoo.service.server.restart()
return {
'type': 'ir.actions.client',
'tag': 'home',
'params': {'wait': True},
}
return post_install_action
finally:
shutil.rmtree(tmp)
input_file = gzip.GzipFile(input_file_name, 'r')
graph = json.loads(input_file.read().decode('utf-8'))
if 'edges' not in graph:
print("ERROR: Input JSON file doesn't have an 'edges' property!",
file=sys.stderr)
else:
edges = graph['edges']
retired_cuis_by_type = get_retired_cuis_by_type()
stats = {
'_report_datetime':
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
'_total_number_of_edges':
len(
edges
), # underscore is to make sure it sorts to the top of the report
'number_of_edges_with_retired_cui':
count_edges_with_cui_in_set(edges,
retired_cuis_by_type.get('ALL')),
'number_of_edges_with_retired_cui_with_synonym':
count_edges_with_cui_in_set(edges, retired_cuis_by_type.get('SY')),
'number_of_edges_with_invalid_cui':
count_edges_with_invalid_cui(edges)
}
temp_output_file = tempfile.mkstemp(prefix='kg2-')[1]
with open(temp_output_file, 'w') as outfile:
json.dump(stats, outfile, indent=4, sort_keys=True)
shutil.move(temp_output_file, args.outputFile[0])
def test_filesystem_notifications(qtbot, projects, tmpdir):
"""
Test that filesystem notifications are emitted when creating,
deleting and moving files and directories.
"""
# Create a directory for the project and some files.
project_root = tmpdir.mkdir('project0')
folder0 = project_root.mkdir('folder0')
folder1 = project_root.mkdir('folder1')
file0 = project_root.join('file0')
file1 = folder0.join('file1')
file2 = folder0.join('file2')
file3 = folder1.join('file3')
file0.write('')
file1.write('')
file3.write('ab')
# Open the project
projects.open_project(path=to_text_string(project_root))
# Get a reference to the filesystem event handler
fs_handler = projects.watcher.event_handler
# Test file creation
with qtbot.waitSignal(fs_handler.sig_file_created,
timeout=30000) as blocker:
file2.write('')
file_created, is_dir = blocker.args
assert file_created == to_text_string(file2)
assert not is_dir
# Test folder creation
with qtbot.waitSignal(fs_handler.sig_file_created,
timeout=3000) as blocker:
folder2 = project_root.mkdir('folder2')
folder_created, is_dir = blocker.args
assert folder_created == osp.join(to_text_string(project_root), 'folder2')
# Test file move/renaming
new_file = osp.join(to_text_string(folder0), 'new_file')
with qtbot.waitSignal(fs_handler.sig_file_moved,
timeout=3000) as blocker:
shutil.move(to_text_string(file1), new_file)
original_file, file_moved, is_dir = blocker.args
assert original_file == to_text_string(file1)
assert file_moved == new_file
assert not is_dir
# Test folder move/renaming
new_folder = osp.join(to_text_string(project_root), 'new_folder')
with qtbot.waitSignal(fs_handler.sig_file_moved,
timeout=3000) as blocker:
shutil.move(to_text_string(folder2), new_folder)
original_folder, folder_moved, is_dir = blocker.args
assert original_folder == to_text_string(folder2)
assert folder_moved == new_folder
assert is_dir
# Test file deletion
with qtbot.waitSignal(fs_handler.sig_file_deleted,
timeout=3000) as blocker:
os.remove(to_text_string(file0))
deleted_file, is_dir = blocker.args
assert deleted_file == to_text_string(file0)
assert not is_dir
assert not osp.exists(to_text_string(file0))
# Test folder deletion
with qtbot.waitSignal(fs_handler.sig_file_deleted,
timeout=3000) as blocker:
shutil.rmtree(to_text_string(folder0))
deleted_folder, is_dir = blocker.args
assert to_text_string(folder0) in deleted_folder
# For some reason this fails in macOS
if not sys.platform == 'darwin':
# Test file/folder modification
with qtbot.waitSignal(fs_handler.sig_file_modified,
timeout=3000) as blocker:
file3.write('abc')
modified_file, is_dir = blocker.args
assert modified_file in to_text_string(file3)
def combine_movie_files(self):
"""
Used internally by Manim to combine the separate
partial movie files that make up a Scene into a single
video file for that Scene.
"""
# Manim renders the scene as many smaller movie files
# which are then concatenated to a larger one. The reason
# for this is that sometimes video-editing is made easier when
# one works with the broken up scene, which effectively has
# cuts at all the places you might want. But for viewing
# the scene as a whole, one of course wants to see it as a
# single piece.
kwargs = {
"remove_non_integer_files": True,
"extension": self.movie_file_extension,
}
if self.scene.start_at_animation_number is not None:
kwargs["min_index"] = self.scene.start_at_animation_number
if self.scene.end_at_animation_number is not None:
kwargs["max_index"] = self.scene.end_at_animation_number
else:
kwargs["remove_indices_greater_than"] = self.scene.num_plays - 1
partial_movie_files = get_sorted_integer_files(
self.partial_movie_directory, **kwargs)
if len(partial_movie_files) == 0:
print("No animations in this scene")
return
# Write a file partial_file_list.txt containing all
# partial movie files
file_list = os.path.join(self.partial_movie_directory,
"partial_movie_file_list.txt")
with open(file_list, 'w') as fp:
for pf_path in partial_movie_files:
if os.name == 'nt':
pf_path = pf_path.replace('\\', '/')
fp.write("file \'file:{}\'\n".format(pf_path))
movie_file_path = self.get_movie_file_path()
commands = [
FFMPEG_BIN,
'-y', # overwrite output file if it exists
'-f',
'concat',
'-safe',
'0',
'-i',
file_list,
'-loglevel',
'error',
'-c',
'copy',
movie_file_path
]
if not self.includes_sound:
commands.insert(-1, '-an')
combine_process = subprocess.Popen(commands)
combine_process.wait()
if self.includes_sound:
sound_file_path = movie_file_path.replace(
self.movie_file_extension, ".wav")
# Makes sure sound file length will match video file
self.add_audio_segment(AudioSegment.silent(0))
self.audio_segment.export(
sound_file_path,
bitrate='312k',
)
temp_file_path = movie_file_path.replace(".", "_temp.")
commands = [
"ffmpeg",
"-i",
movie_file_path,
"-i",
sound_file_path,
'-y', # overwrite output file if it exists
"-c:v",
"copy",
"-c:a",
"aac",
"-b:a",
"320k",
# select video stream from first file
"-map",
"0:v:0",
# select audio stream from second file
"-map",
"1:a:0",
'-loglevel',
'error',
# "-shortest",
temp_file_path,
]
subprocess.call(commands)
shutil.move(temp_file_path, movie_file_path)
os.remove(sound_file_path)
self.print_file_ready_message(movie_file_path)
