def justThree():
print("One directory. Piece of cake after you answer some questions")
valdir = False # Valid directory input
filename = "";
while not valdir:
filename = input("Input the directory name: ")
if ntpath.isdir(filename):
valdir = True
else:
print("That is not an valid directory. Try again")
# Creating Target Destination
replace = input("Are you going to be replacing the file? ")
replacing = yesNo(replace);
newPath = "";
if not replacing:
newPath = input("What are we going to call this new folder destination ")
if not os.path.exists(newPath):
os.mkdir(newPath)
else:
print("What a joy. Just replacing")
# Gets files in the directory
filenames = shallowPath(filename);
# Cleans Files
print("Now it is time for me to get to work")
con = 0;
for fil in filenames:
print('Cleaning ' + os.path.basename(fil))
con += 1
if replacing:
getOn(fil, fil)
else:
getOn(fil, newPath + "\\" + (stripFileName(fil)))
print("Ha. I finished.\nHave a good day with your " + str(con) + " Cleaned files")
def justFive():
print("One directory and their children. That is just great")
valdir = False # Valid directory input
filename = "";
while not valdir:
filename = input("Input the directory name: ")
if ntpath.isdir(filename):
valdir = True
else:
print("That is not an valid directory. Try again")
# Creating Target Destination
replace = input("Are you going to be replacing the file? ")
replacing = yesNo(replace);
newPath = "";
if not replacing:
newPath = input("What is the name of this destination folder ")
if not os.path.exists(newPath):
os.mkdir(newPath)
else:
print("Perfect. Just replacing")
# Gets files in the directory
filenames = allThePaths(filename);
# Cleans Files
print("Now lets get down to the good stuff")
con = 0;
for fil in filenames:
print('Cleaning ' + os.path.basename(fil))
con += 1
if replacing:
getOn(fil, fil)
else:
getOn(fil, newPath + "\\" + (stripFileName(fil)))
print("I have finished.\nHave a good day with your " + str(con) + " Cleaned files")
def _bundle_zip(tmpfile, bundle_type, environment, paths):
""" Create a zip archive
:type tmpfile: tempfile instance
:param tmpfile: Tempfile object
:type bundle_type: str
:param bundle_type: Bundle name
:type environment: str
:param environment: Environment name
:type paths: list
:param paths: List of paths to include
"""
logger.info('Generating zip file for {}'.format(bundle_type))
archive = zipfile.ZipFile(tmpfile, 'w')
path_rewrites = config.get_bundle_path_rewrites(bundle_type)
for path in paths:
path = _convert_paths_to_local_format(path)
if ospath.isdir(path):
# Extract all file names from directory
filenames = _find_files(path)
else:
filenames = [path]
for filename in filenames:
arcname = filename
# Exclude files with other target environments
prefix = '__cumulus-{}__'.format(environment)
basename = ospath.basename(filename)
if basename.startswith('__cumulus-'):
if len(basename.split(prefix)) != 2:
logger.debug('Excluding file {}'.format(filename))
continue
elif prefix in filename.split(ospath.sep):
logger.debug('Excluding file {}'.format(filename))
continue
# Do all rewrites
for rewrite in path_rewrites:
target = _convert_paths_to_local_format(
rewrite['target'].replace('\\\\', '\\'))
destination = _convert_paths_to_local_format(
rewrite['destination'].replace('\\\\', '\\'))
try:
if arcname[:len(target)] == target:
arcname = arcname.replace(target, destination)
logger.debug(
'Replaced "{}" with "{}" in bundle {}'.format(
target, destination, bundle_type))
except IndexError:
pass
logger.debug('Adding: {}'.format(filename))
archive.write(filename, arcname, zipfile.ZIP_DEFLATED)
archive.close()
def __init__(self, pathname):
assert (ntpath.isdir(pathname) or ntpath.isfile(pathname)), "no file or directory %s exists" % pathname
self.pathname = pathname
self.audiofiles = []
self.artwork = None
self.clr = False
if ntpath.isfile(pathname):
extension = ntpath.basename(pathname).rsplit('.', 1)[-1]
assert extension == "mp3", "file must be mp3 format"
self.audiofiles.append(pathname)
if ntpath.isdir(pathname):
dir_files = utl.get_multiple(pathname, "mp3")
assert dir_files, "No mp3 files in given directory"
self.audiofiles.extend(dir_files)
def get_file(self, extended_path, revision=HEAD, **kwargs):
"""Return content of file or list content of directory"""
if not extended_path:
raise FileNotFoundError(extended_path, revision)
if revision == PRE_CREATION:
return ''
if self.viewtype == self.VIEW_SNAPSHOT:
# Get the path to (presumably) file element (remove version)
# The '@@' at the end of file_path is required.
file_path = extended_path.rsplit('@@', 1)[0] + '@@'
okind = self._get_object_kind(file_path)
if okind == 'directory element':
raise SCMError('Directory elements are unsupported.')
elif okind == 'file element':
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
else:
if cpath.isdir(extended_path):
output = self.client.list_dir(extended_path, revision)
elif cpath.exists(extended_path):
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
return output
def makeFolder():
## Normalize input
global inputDataPath, workingFolder, fileName, fileName, ext, startTime
global saveFolder, logsFileName, outputFileName, serializeFileName
global outputDataPath, logsDataPath, serializeDataPath
inputDataPath = ntpath.normcase(inputDataPath)
## Folder/File/Ext Names
workingFolder, fileName = ntpath.split(inputDataPath)
fileName, ext = ntpath.splitext(fileName)
# startTime = time.strftime("%x_%X") <= probleme sur win avec : dans le nom de fichier
startTime = time.strftime("%Y-%m-%d_%Hh%Mm%Ss")
saveFolder = ntpath.join(workingFolder, fileName) + "Results"
logsFileName = startTime + "_Logs"
outputFileName = startTime + "_Load"
serializeFileName = startTime + "_Serialize"
outputDataPath = ntpath.join(saveFolder, logsFileName)
logsDataPath = ntpath.join(saveFolder, outputFileName)
serializeDataPath = ntpath.join(saveFolder, serializeFileName)
## Make dir
if not ntpath.isdir(saveFolder):
os.makedirs(saveFolder)
a = open(outputDataPath, 'w+')
a = open(logsDataPath, 'w+')
def _set_label(self, label, path):
"""Set a ClearCase label on elements seen under path.
Args:
label (unicode):
The label to set.
path (unicode):
The filesystem path to set the label on.
"""
checkedout_elements = self._list_checkedout(path)
if checkedout_elements:
raise Exception(
'ClearCase backend cannot set label when some elements are '
'checked out:\n%s' % ''.join(checkedout_elements))
# First create label in vob database.
execute(['cleartool', 'mklbtype', '-c', 'label created for rbtools',
label],
with_errors=True)
# We ignore return code 1 in order to omit files that ClearCase cannot
# read.
recursive_option = ''
if cpath.isdir(path):
recursive_option = '-recurse'
# Apply label to path.
execute(['cleartool', 'mklabel', '-nc', recursive_option, label, path],
extra_ignore_errors=(1,),
with_errors=False)
def get_file(self, extended_path, revision=HEAD):
"""Return content of file or list content of directory"""
if not extended_path:
raise FileNotFoundError(extended_path, revision)
if revision == PRE_CREATION:
return ''
if self.viewtype == self.VIEW_SNAPSHOT:
# Get the path to (presumably) file element (remove version)
# The '@@' at the end of file_path is required.
file_path = extended_path.rsplit('@@', 1)[0] + '@@'
okind = self._get_object_kind(file_path)
if okind == 'directory element':
raise SCMError('Directory elements are unsupported.')
elif okind == 'file element':
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
else:
if cpath.isdir(extended_path):
output = self.client.list_dir(extended_path, revision)
elif cpath.exists(extended_path):
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
return output
def _set_label(self, label, path):
"""Set a ClearCase label on elements seen under path."""
checkedout_elements = self._list_checkedout(path)
if checkedout_elements:
raise Exception(
'ClearCase backend cannot set label when some elements are '
'checked out:\n%s' % ''.join(checkedout_elements))
# First create label in vob database.
execute([
'cleartool', 'mklbtype', '-c', 'label created for rbtools', label
],
with_errors=True)
# We ignore return code 1 in order to omit files that ClearCase cannot
# read.
recursive_option = ''
if cpath.isdir(path):
recursive_option = '-recurse'
# Apply label to path.
execute(['cleartool', 'mklabel', '-nc', recursive_option, label, path],
extra_ignore_errors=(1, ),
with_errors=False)
def _get_rule_files(rules_folder, rule):
if not isinstance(rules_folder, str) and not isinstance(rule, str):
raise TypeError(
"Error: You must set a path containing rule files"
" in rules_folder or the path to a rule file in rule.")
rule_files = {}
if rule is not None:
try:
yara.compile(filepath=rule)
except yara.Error as er:
utils.display_warning("skipping rule {}.\n{}".format(rule, er))
return None
rule_files[ntpath.basename(rule)] = rule
else:
if ntpath.isdir(rules_folder):
for file in os.listdir(rules_folder):
rule = ntpath.join(rules_folder, file)
if ntpath.isfile(rule) and file not in rule_files:
try:
yara.compile(filepath=rule)
rule_files[file] = rule
except yara.Error as er:
utils.display_warning(
"skipping rule {}.\n{}".format(rule, er))
return rule_files
def makedirs(name, mode=0o777, exist_ok=False):
"""makedirs(name [, mode=0o777][, exist_ok=False])
Super-mkdir; create a leaf directory and all intermediate ones. Works like
mkdir, except that any intermediate path segment (not just the rightmost)
will be created if it does not exist. If the target directory already
exists, raise an OSError if exist_ok is False. Otherwise no exception is
raised. This is recursive.
"""
head, tail = path.split(name)
if not tail:
head, tail = path.split(head)
if head and tail and not path.exists(head):
try:
makedirs(head, mode, exist_ok)
except FileExistsError:
# Defeats race condition when another thread created the path
pass
cdir = curdir
if isinstance(tail, bytes):
cdir = bytes(curdir, 'ASCII')
if tail == cdir: # xxx/newdir/. exists if xxx/newdir exists
return
try:
mkdir(name, mode)
except OSError:
# Cannot rely on checking for EEXIST, since the operating system
# could give priority to other errors like EACCES or EROFS
if not exist_ok or not path.isdir(name):
raise
def ensure_dir_exist(directory):
if not ntpath.exists(directory):
print("Directory '{}' does not exists".format(directory))
exit(-1)
if not ntpath.isdir(directory):
print("Directory name '{}' is not a directory".format(directory))
exit(-1)
def get_file(self, extended_path, revision=HEAD, **kwargs):
"""Return content of file or list content of directory.
Args:
extended_path (unicode):
The path of the element, including revision information.
revision (reviewboard.scmtools.core.Revision, optional):
Revision information. This will be either
:py:data:`~reviewboard.scmtools.core.PRE_CREATION` (new file),
or :py:data:`~reviewboard.scmtools.core.HEAD` (signifying to
use the revision information included in ``extended_path``).
**kwargs (dict, optional):
Additional unused keyword arguments.
Returns:
bytes:
The contents of the element.
Raises:
reviewboard.scmtools.errors.FileNotFoundError:
The given ``extended_path`` did not match a valid element.
reviewboard.scmtools.errors.SCMError:
Another error occurred.
"""
if not extended_path:
raise FileNotFoundError(extended_path, revision)
if revision == PRE_CREATION:
return ''
if self.viewtype == self.VIEW_SNAPSHOT:
# Get the path to (presumably) file element (remove version)
# The '@@' at the end of file_path is required.
file_path = extended_path.rsplit('@@', 1)[0] + '@@'
okind = self._get_element_kind(file_path)
if okind == 'directory element':
raise SCMError('Directory elements are unsupported.')
elif okind == 'file element':
output = self.client.cat_file(extended_path)
else:
raise FileNotFoundError(extended_path)
else:
if cpath.isdir(extended_path):
output = self.client.list_dir(extended_path)
elif cpath.exists(extended_path):
output = self.client.cat_file(extended_path)
else:
raise FileNotFoundError(extended_path)
return output
def save_excel(self):
if os.name == 'posix':
self.wb.save('{}.xlsx'.format(datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M")))
sys.exit('文件已经保存为 --> {}.xlsx'.format(datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M")))
else:
save_position = input('要保存到哪个盘符下: ')
if ntpath.isdir('{}:'.format(save_position)):
self.wb.save(r'{0}:\{1}.xlsx'.format(save_position, datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M")))
sys.exit('文件已经保存到 --> {0}:\{1}.xlsx'.format(save_position,
datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M")))
else:
print('该盘符不存在,请重新输入!')
self.save_excel()
def __init__(self, pathname):
assert (ntpath.isdir(pathname) or ntpath.isfile(pathname)
), "no file or directory %s exists" % pathname
self.pathname = pathname
self.videofiles = {}
if ntpath.isfile(pathname):
filename, extension = ntpath.basename(pathname).rsplit('.', 1)
assert extension == "mp4", "file must be mp4 format"
self.videofiles[pathname] = filename
self.directory = ntpath.dirname(pathname)
if ntpath.isdir(pathname):
dir_files = utl.get_multiple(pathname, "mp4")
assert dir_files, "no mp4 files in given directory"
for file in dir_files:
filename, extension = ntpath.basename(file).rsplit('.', 1)
self.videofiles[file] = filename
self.directory = pathname
self.artwork = None
self.clr = False
def _diff(self, old_file, new_file, xpatches=None, unified=True):
"""Calculate the diff.
Content should be a list of strings with no endl. Supports
exclude patches (list of list of strings with no endl). If the
content is None, it is assumed that the file is binary. The file
names (new_file and old_file) are only to be used as a header
for a diff.
Returns None if the files are different and binary. Otherwise
returns a difference as a list of strings with no lineseps. The
binary files which are equal also return an empty string."""
old_content = None
new_content = None
# The content should have line endings removed from it!
if cpath.isdir(new_file):
# read directory content
old_content = sorted(os.listdir(old_file)) + ['']
new_content = sorted(os.listdir(new_file)) + ['']
elif cpath.exists(new_file):
# returns None for binary file
old_content = read_text_file(old_file)
new_content = read_text_file(new_file)
else:
logging.debug("File %s does not exist or access is denied."
% new_file)
return None
# check if binary files and if they differ
if old_content is None or new_content is None:
old_crc = zlib.crc32(open(old_file).read())
new_crc = zlib.crc32(open(new_file).read())
if old_crc != new_crc:
return None
else:
return u''
# check if we need to exclude anything from the diff
if xpatches:
for patch in reversed(xpatches):
patched = self._patch(new_content, patch)
if patched:
new_content = patched
return self._content_diff(old_content, new_content, old_file,
new_file, unified=unified)
def _remove_old_files():
""" Remove files from previous bundle """
cache_file = '/var/local/cumulus-bundle-handler.cache'
if sys.platform in ['win32', 'cygwin']:
if not ospath.exists('C:\\cumulus\\cache'):
os.makedirs('C:\\cumulus\\cache')
cache_file = 'C:\\cumulus\\cache\\cumulus-bundle-handler.cache'
if not ospath.exists(cache_file):
LOGGER.info('No previous bundle files to clean up')
return
LOGGER.info('Removing old files and directories')
with open(cache_file, 'r') as file_handle:
for line in file_handle.readlines():
line = line.replace('\n', '')
if not ospath.exists(line):
continue
if ospath.isdir(line):
try:
os.removedirs(line)
LOGGER.debug('Removing directory {}'.format(line))
except OSError:
pass
elif ospath.isfile(line):
LOGGER.debug('Removing file {}'.format(line))
os.remove(line)
try:
os.removedirs(ospath.dirname(line))
except OSError:
pass
elif ospath.islink(line):
LOGGER.debug('Removing link {}'.format(line))
os.remove(line)
try:
os.removedirs(ospath.dirname(line))
except OSError:
pass
else:
LOGGER.warning('Unknown file type {}'.format(line))
# Remove the cache file when done
os.remove(cache_file)
def _remove_old_files():
""" Remove files from previous bundle """
cache_file = '/var/local/cumulus-bundle-handler.cache'
if sys.platform in ['win32', 'cygwin']:
if not ospath.exists('C:\\cumulus\\cache'):
os.makedirs('C:\\cumulus\\cache')
cache_file = 'C:\\cumulus\\cache\\cumulus-bundle-handler.cache'
if not ospath.exists(cache_file):
LOGGER.info('No previous bundle files to clean up')
return
LOGGER.info('Removing old files and directories')
with open(cache_file, 'r') as file_handle:
for line in file_handle.readlines():
line = line.replace('\n', '')
if not ospath.exists(line):
continue
if ospath.isdir(line):
try:
os.removedirs(line)
LOGGER.debug('Removing directory {}'.format(line))
except OSError:
pass
elif ospath.isfile(line):
LOGGER.debug('Removing file {}'.format(line))
os.remove(line)
try:
os.removedirs(ospath.dirname(line))
except OSError:
pass
elif ospath.islink(line):
LOGGER.debug('Removing link {}'.format(line))
os.remove(line)
try:
os.removedirs(ospath.dirname(line))
except OSError:
pass
else:
LOGGER.warning('Unknown file type {}'.format(line))
# Remove the cache file when done
os.remove(cache_file)
def get_file(self, extended_path, revision=HEAD):
"""Return content of file or list content of directory"""
if not extended_path:
raise FileNotFoundError(extended_path, revision)
if revision == PRE_CREATION:
return ''
if cpath.isdir(extended_path):
output = self.client.list_dir(extended_path, revision)
elif cpath.exists(extended_path):
output = self.client.cat_file(extended_path, revision)
else:
raise FileNotFoundError(extended_path, revision)
return output
def _do_diff(self, changeset):
"""Generate a unified diff for all files in the given changeset.
Args:
changeset (list):
A list of changes.
Returns:
dict:
A dictionary containing a ``diff`` key.
"""
# Sanitize all changesets of version 0 before processing
changeset = self._sanitize_version_0_changeset(changeset)
diff = []
for old_file, new_file in changeset:
dl = []
# cpath.isdir does not work for snapshot views but this
# information can be found using `cleartool describe`.
if self.viewtype == 'snapshot':
# ClearCase object path is file path + @@
object_path = new_file.split('@@')[0] + '@@'
output = execute(['cleartool', 'describe', '-fmt', '%m',
object_path])
object_kind = output.strip()
isdir = object_kind == 'directory element'
else:
isdir = cpath.isdir(new_file)
if isdir:
dl = self._diff_directories(old_file, new_file)
elif cpath.exists(new_file) or self.viewtype == 'snapshot':
dl = self._diff_files(old_file, new_file)
else:
logging.error('File %s does not exist or access is denied.',
new_file)
continue
if dl:
diff.append(b''.join(dl))
return {
'diff': b''.join(diff),
}
def do_diff(self, changeset):
"""Generates a unified diff for all files in the changeset."""
diff = []
for old_file, new_file in changeset:
dl = []
if cpath.isdir(new_file):
dl = self.diff_directories(old_file, new_file)
elif cpath.exists(new_file):
dl = self.diff_files(old_file, new_file)
else:
logging.error("File %s does not exist or access is denied."
% new_file)
continue
if dl:
diff.append(''.join(dl))
return (''.join(diff), None)
def download_and_unpack_bundle(bundle_type):
""" Download the bundle from AWS S3
:type bundle_type: str
:param bundle_type: Bundle type to download
"""
key = _get_key(bundle_type)
# If the bundle does not exist
if not key:
LOGGER.error('No bundle found. Exiting.')
sys.exit(0)
bundle = tempfile.NamedTemporaryFile(suffix='.zip', delete=False)
bundle.close()
LOGGER.info("Downloading s3://{}/{} to {}".format(
config.get('bundle-bucket'), key.name, bundle.name))
key.get_contents_to_filename(bundle.name)
extraction_path = _get_extraction_path(bundle_type)
# Unpack the bundle
archive = zipfile.ZipFile(bundle.name, 'r')
_store_bundle_files(archive.namelist(), extraction_path)
try:
LOGGER.info('Unpacking {} to {}'.format(bundle.name, extraction_path))
archive.extractall(extraction_path)
for info in archive.infolist():
archive.extract(info, extraction_path)
if not ospath.isdir(ospath.join(extraction_path, info.filename)):
os.chmod(ospath.join(extraction_path, info.filename),
info.external_attr >> 16)
except Exception as err:
LOGGER.error('Error when unpacking bundle: {}'.format(err))
finally:
archive.close()
# Remove the downloaded package
LOGGER.info("Removing temporary file {}".format(bundle.name))
os.remove(bundle.name)
def parse_args(argv, name=None):
download_dir = CFG.installers_base_dir
try:
opts, args = getopt.getopt(argv, "ho:l:",
["output-dir=", "limit="])
except getopt.GetoptError:
Crawler.usage()
sys.exit(2)
# Default value
limit = LIMIT
for opt, arg in opts:
if opt == '-h':
Crawler.usage()
sys.exit()
elif opt in ("-l", "--limit"):
try:
limit = int(arg)
except:
Crawler.usage()
sys.exit(4)
# If no outputdir has been specified, crash.
if download_dir is None:
sys.stderr.write("Invalid or missing outputdir specified.")
sys.exit(3)
else:
# Create the directory if doesn't exist
if not ntpath.isdir(download_dir):
os.mkdir(download_dir)
# Create a subdir for this run
if name is None:
name = str(time.time())
download_dir = os.path.join(download_dir, name)
if not os.path.isdir(download_dir):
os.mkdir(download_dir)
return limit, download_dir
def _do_diff(self, changeset):
"""Generates a unified diff for all files in the changeset."""
# Sanitize all changesets of version 0 before processing
changeset = self._sanitize_version_0_changeset(changeset)
diff = []
for old_file, new_file in changeset:
dl = []
# cpath.isdir does not work for snapshot views but this
# information can be found using `cleartool describe`.
if self.viewtype == 'snapshot':
# ClearCase object path is file path + @@
object_path = new_file.split('@@')[0] + '@@'
output = execute(
["cleartool", "describe", "-fmt", "%m", object_path])
object_kind = output.strip()
isdir = object_kind == 'directory element'
else:
isdir = cpath.isdir(new_file)
if isdir:
dl = self._diff_directories(old_file, new_file)
elif cpath.exists(new_file) or self.viewtype == 'snapshot':
dl = self._diff_files(old_file, new_file)
else:
logging.error("File %s does not exist or access is denied." %
new_file)
continue
if dl:
diff.append(''.join(dl))
return {
'diff': ''.join(diff),
}
def makeFolder(self):
## Normalize input
self.inputDataPath = ntpath.normcase(self.inputDataPath)
## Folder/File/Ext Names
self.workingFolder, self.fileName = ntpath.split(self.inputDataPath)
self.fileName, self.ext = ntpath.splitext(self.fileName)
# self.startTime = time.strftime("%x_%X") <= probleme sur win avec : dans le nom de fichier
self.startTime = time.strftime("%Y-%m-%d_%Hh%Mm%Ss")
self.saveFolder = ntpath.join(self.workingFolder, self.fileName) + "Results"
self.logsFile = self.startTime + "_Logs"
self.outputFile = self.startTime + "_Load"
self.outputDataPath = ntpath.join(self.saveFolder, self.logsFile)
self.logDataPath = ntpath.join(self.saveFolder, self.outputFile)
## Make dir
if not ntpath.isdir(self.saveFolder):
os.makedirs(self.saveFolder)
test = open(self.outputDataPath, 'w+')
self.debugAll()
self.train(self.inputDataPath)
def _do_diff(self, changeset):
"""Generates a unified diff for all files in the changeset."""
# Sanitize all changesets of version 0 before processing
changeset = self._sanitize_version_0_changeset(changeset)
diff = []
for old_file, new_file in changeset:
dl = []
# cpath.isdir does not work for snapshot views but this
# information can be found using `cleartool describe`.
if self.viewtype == 'snapshot':
# ClearCase object path is file path + @@
object_path = new_file.split('@@')[0] + '@@'
output = execute(["cleartool", "describe", "-fmt", "%m",
object_path])
object_kind = output.strip()
isdir = object_kind == 'directory element'
else:
isdir = cpath.isdir(new_file)
if isdir:
dl = self._diff_directories(old_file, new_file)
elif cpath.exists(new_file) or self.viewtype == 'snapshot':
dl = self._diff_files(old_file, new_file)
else:
logging.error("File %s does not exist or access is denied."
% new_file)
continue
if dl:
diff.append(''.join(dl))
return {
'diff': ''.join(diff),
}
def arguments():
try:
opts, args = getopt.getopt(sys.argv[1:], "hi:o:v",
["help", "input=", "output=", "verbose"])
except getopt.GetoptError as err:
## Error in arguments
print str(err)
usage()
sys.exit(2)
## Default arguments
inputDataPath = 'example.txt'
outputDataPath = 'exampleOutput.txt'
logDataPath = 'exampleLog.txt'
verbose = False
## Mandatory
inputDataPathFound = False
outputDataPathFound = False
logDataPathFound = False
#print opts
for o, a in opts:
if o in ("-v", "--verbose"):
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit(0)
elif o in ("-i", "--input"):
inputDataPath = a
inputDataPathFound = True
#elif o in ("-o", "--output"):
#outputDataPath = a
#outputDataPathFound = True
#elif o in ("-l", "--log"):
#logDataPath = a
#logDataPathFound = True
else:
assert False, "unhandled option"
## Mandatory Check
if not inputDataPathFound: ## Is an input given?
print "-i [file_path] or --input [file_path] was not given"
usage()
sys.exit(2)
if not ntpath.isfile(inputDataPath): ## Is input a file?
print "The path given as an input is not a file"
usage()
sys.exit(2)
## Folder/File/Ext Names
workingFolder, fileName = ntpath.split(inputDataPath)
#fileName = ntpath.basename(inputDataPath) if '.' not in ntpath.basename(inputDataPath) else ntpath.basename(inputDataPath)
fileName, ext = ntpath.splitext(fileName)
#startTime = time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime())
startTime = time.strftime("%x_%X")
saveFolder = ntpath.join(workingFolder, fileName) + "Results"
logsFile = startTime + "_Logs"
loadFile = startTime + "_Load"
if verbose:
print "Starting Time : " + startTime
print "workingFolder : " + workingFolder
print "saveFolder : " + saveFolder
print "fileName : " + fileName
print "ext : " + ext
## Make dir
if not ntpath.isdir(saveFolder):
os.makedirs(saveFolder)
train(inputDataPath)
raise OSError(errno.EEXIST, errno.strerror(errno.EEXIST),
filename)
except java.io.IOException, ioe:
raise OSError(ioe)
mode = '%s%s%s%s' % (reading and 'r' or '',
(not appending and writing) and 'w' or '',
(appending and (writing or updating)) and 'a' or '',
updating and '+' or '')
if sync and (writing or updating):
from java.io import FileNotFoundException, RandomAccessFile
try:
fchannel = RandomAccessFile(sys.getPath(filename), 'rws').getChannel()
except FileNotFoundException, fnfe:
if path.isdir(filename):
raise OSError(errno.EISDIR, errno.strerror(errno.EISDIR))
raise OSError(errno.ENOENT, errno.strerror(errno.ENOENT), filename)
return FileIO(fchannel, mode)
return FileIO(filename, mode)
def read(fd, buffersize):
"""read(fd, buffersize) -> string
Read a file descriptor.
"""
from org.python.core.util import StringUtil
rawio = FileDescriptors.get(fd)
buf = _handle_oserror(rawio.read, buffersize)
return asPyString(StringUtil.fromBytes(buf))
def update_event(self, inp=-1):
self.set_output_val(0, ntpath.isdir(self.input(0)))
def main(argv):
global totalFileSize
# Parameter 1 - Import Directory
# Parameter 2 - Subfolder Level
# Remember the start time of the program
start_time = time.clock()
usage_string = " DocumentLoader.py -s <src> -r <recursive Level> -d <oracle_directory_path> -c <config_path>"
path_name = '-'
dest_name = '-'
recursiveLevel = 0
config_path = 'dataLoader.conf'
# -- Parameter from the config file
oracle_port = '-'
oracle_host = '-'
oracle_service = '-'
oracle_user = '******'
oracle_pwd = '-'
magicFilePath = '-'
ingoreFileExt = []
# Path of the oracle Info Archive
ora_dir_path = '-'
try:
opts, args = getopt.getopt(argv, "hs:d:r:d:c:", ["src=", "rec=", "dir=", "config="])
except getopt.GetoptError:
print("usage: {0}".format(usage_string))
sys.exit(2)
# read the parameter
for opt, arg in opts:
if opt == '-h':
print("usage: {0}").format(usage_string)
sys.exit()
elif opt in ("-s", "--src"):
path_name = arg
elif opt in ("-d", "--dir"):
ora_dir_path = arg
elif opt in ("-c", "--config"):
config_path = arg
elif opt in ("-r", "--rec"):
recursiveLevel = int(arg)
# read the config file
config = configparser.ConfigParser()
# check if the file exits
if os.path.exists(config_path) == False:
# use normal configparser to write the template
print("--" + 80 * "!")
print("-- Error to read file {0}".format(config_path))
print("-- Error usage: {0}".format(usage_string))
config['DEFAULT'] = {'MagicFile': 'd:\\tools\\file\\bin\\file.exe', 'ignoreFileExt': '.iso'}
config['ORACLE_DB_CONNECT'] = {'Host': 'localhost', 'Port': '1521', 'Service': 'ORCL', 'DB_User': '******',
'DB_Password': '******'}
with open(config_path, 'w') as configfile:
config.write(configfile)
print("--" + 80 * "!")
print("-- Info create Configuration Template :: {0}".format(config_path))
print("-- Info fillout the configuration file with your personal values and start again!")
print("--" + 80 * "!")
sys.exit(2)
else:
print("-- Info read config file {0}".format(config_path))
config.read(config_path)
# Parameter of the application
general_configuration = config['DEFAULT']
magicFilePath = general_configuration['MagicFile']
ingoreFileExtString = general_configuration['ignoreFileExt']
ingoreFileExt = str.split(str.replace(ingoreFileExtString, ' ', ''), ',')
# Oracle DB Connect
oracle_db_configuration = config['ORACLE_DB_CONNECT']
oracle_port = oracle_db_configuration['Port']
oracle_host = oracle_db_configuration['Host']
oracle_service = oracle_db_configuration['Service']
oracle_user = oracle_db_configuration['DB_User']
oracle_pwd = oracle_db_configuration['DB_Password']
# check if Directory exists and if the * is necessary
# BUG ! if more then 1 then the * not match the documents on root level??
# FIX IT!
# Source
if os.path.isdir(path_name):
if path_name.endswith(os.path.sep):
path_name += ("*" + os.path.sep) * recursiveLevel
path_name += "*.*"
else:
path_name += os.path.sep
path_name += ("*" + os.path.sep) * recursiveLevel
path_name += "*.*"
else:
print("-- Error :: 05 Source Directory (-s) {0} not found".format(path_name))
print("usage: {0}").format(usage_string)
sys.exit(2)
# connect to the database
print("--" + 40 * "=")
print("-- Info :: Oracle Client Library Version :: {0}".format(cx_Oracle.clientversion()))
# get the connection to the database
print("-- Info :: oracle_host {0} oracle_port {1} oracle_service {2} oracle_user {3} oracle_pwd ********".format(
oracle_host, oracle_port, oracle_service, oracle_user, oracle_pwd))
connection = cx_Oracle.connect(
oracle_user + '/' + oracle_pwd + '@' + oracle_host + ':' + oracle_port + '/' + oracle_service)
# Version der DB ausgeben
print("-- Info :: Oracle Database Version :: {0}".format(connection.version))
print("--" + 40 * "=")
print("-- Info :: Environment Settings :: Language :: {0} - Char Set ::{1}".format(locale.getdefaultlocale()[0],
locale.getdefaultlocale()[1]))
print("--" + 40 * "=")
print("-- Info :: Read all files from {0}".format(path_name))
print("-- Info :: Copy files to {0}".format(dest_name))
print("-- Info :: Not index this file types ::" + str(ingoreFileExt))
print("--" + 40 * "=")
fileCount = 0
fileExistsCount = 0
dirCount = 0
dirPathList = []
totalFileSize = 0
# Get the list of all Files
fileList = glob.glob(path_name)
# remove Thumbs.db if exist from the list
thumbsDBFile = "Thumbs.db"
for file in fileList:
if file.endswith(thumbsDBFile):
fileList.remove(file)
# Loop one read files in Import Directory
for file in fileList:
fileCount += 1
#read only some files
if fileCount > 20:
exit
# do the work
try:
# check if directoy
if ntpath.isdir(file):
print("-- Info :: found dirctory {0} ::".format(file))
dirCount += 1
else:
# get only the filename without the path
filename = ntpath.basename(file)
# get directory
fileDirectoryName = ntpath.dirname(file)
# get Create date
fileAccessDate = datetime.datetime.fromtimestamp(ntpath.getatime(file))
fileModDate = datetime.datetime.fromtimestamp(ntpath.getmtime(file))
fileCreateDate = datetime.datetime.fromtimestamp(ntpath.getctime(file))
# get md5 hash
md5checkSum = getMD5(file)
# get File Type over file from external, Python Lib magic not working, error with magic file!
# now I implement this stupid solution
fileType = getFileType(file, " ", magicFilePath)
# Call file with -i to get the mime Type
fileMimeType = getFileType(file, "-i", magicFilePath)
# get Extenstion
fileExt = getFile_ext(filename)
# getFileSize
fileSize = os.path.getsize(file)
# Remember for statistic
setStatisticTotalSize(fileSize)
# not add url files to the index
# endswith(".url")
if fileExt in ingoreFileExt:
# encode the output with UTF-8 to avoid errors with stange things in filenames
print("-- Info :: Not index this file types ::" + str(ingoreFileExt))
print("-- Info :: Not index this file::{0}".format(repr(filename.encode('utf-8'))))
print("-- Info :: Not index this Dir::{0}".format(repr(fileDirectoryName.encode('utf-8'))))
print("-- --")
else:
# record
fileInfo = dict(filename=str(filename), filepath=str(fileDirectoryName), fileADate=fileAccessDate,
fileMDate=fileModDate, fileCDate=fileCreateDate, md5=md5checkSum,
fileBType=fileType,
fileMtype=fileMimeType, fileExtention=fileExt, fileSize=fileSize)
##print("-- Index this Dir::{0}".format(repr(fileDirectoryName.encode('utf-8'))))
# encode the output with UTF-8 to avoid errors with stange things in filenames
##print("-- Index this file ::{0}".format(repr(filename.encode('utf-8'))))
# write to DB
insertFileInfo(fileInfo, ora_dir_path, connection)
except OSError as exception:
if exception.errno != errno.EEXIST:
print("-- Error :: Error read file :: see error {1}".format(file, sys.exc_info()[0]))
# print statistics
print("--" + 40 * "=")
print("-- Finish with :: {0} files in {1} new directories".format(fileCount, dirCount))
print("-- The run needs :: {0:5.4f} seconds".format(time.clock() - start_time))
print("-- Read size :: {0:5.3f} MB".format(totalFileSize / 1024 / 1024))
print("--" + 40 * "=")
# Close the DB Connection
connection.close()
def fix_symbol_store_path(symbol_store_path=None,
remote=True,
force=False):
"""
Fix the symbol store path. Equivalent to the C{.symfix} command in
Microsoft WinDbg.
If the symbol store path environment variable hasn't been set, this
method will provide a default one.
@type symbol_store_path: str or None
@param symbol_store_path: (Optional) Symbol store path to set.
@type remote: bool
@param remote: (Optional) Defines the symbol store path to set when the
C{symbol_store_path} is C{None}.
If C{True} the default symbol store path is set to the Microsoft
symbol server. Debug symbols will be downloaded through HTTP.
This gives the best results but is also quite slow.
If C{False} the default symbol store path is set to the local
cache only. This prevents debug symbols from being downloaded and
is faster, but unless you've installed the debug symbols on this
machine or downloaded them in a previous debugging session, some
symbols may be missing.
If the C{symbol_store_path} argument is not C{None}, this argument
is ignored entirely.
@type force: bool
@param force: (Optional) If C{True} the new symbol store path is set
always. If C{False} the new symbol store path is only set if
missing.
This allows you to call this method preventively to ensure the
symbol server is always set up correctly when running your script,
but without messing up whatever configuration the user has.
Example::
from winappdbg import Debug, System
def simple_debugger( argv ):
# Instance a Debug object
debug = Debug( MyEventHandler() )
try:
# Make sure the remote symbol store is set
System.fix_symbol_store_path(remote = True,
force = False)
# Start a new process for debugging
debug.execv( argv )
# Wait for the debugee to finish
debug.loop()
# Stop the debugger
finally:
debug.stop()
@rtype: str or None
@return: The previously set symbol store path if any,
otherwise returns C{None}.
"""
try:
if symbol_store_path is None:
local_path = "C:\\SYMBOLS"
if not ntpath.isdir(local_path):
local_path = "C:\\Windows\\Symbols"
if not ntpath.isdir(local_path):
local_path = ntpath.abspath(".")
if remote:
symbol_store_path = (
"cache*;SRV*" + local_path + "*"
"http://msdl.microsoft.com/download/symbols")
else:
symbol_store_path = "cache*;SRV*" + local_path
previous = os.environ.get("_NT_SYMBOL_PATH", None)
if not previous or force:
os.environ["_NT_SYMBOL_PATH"] = symbol_store_path
return previous
except Exception as e:
warnings.warn("Cannot fix symbol path, reason: %s" % str(e),
RuntimeWarning)
def do_diff(self, params):
# Diff returns "1" if differences were found.
# Add the view name and view type to the description
o = []
Feol = False
while len(params) > 0:
# Read both original and modified files.
onam = params.pop(0)
mnam = params.pop(0)
file_data = []
do_rem = False
# If the filename length is greater than 254 char for windows,
# we copied the file to a temp file
# because the open will not work for path greater than 254.
# This is valid for the original and
# modified files if the name size is > 254.
for filenam in (onam, mnam):
if cpath.exists(filenam) and self.viewtype == "dynamic":
do_rem = False
fn = filenam
elif len(filenam) > 254 or self.viewtype == "snapshot":
fn = self.get_filename_hash(filenam)
fn = cpath.join(tempfile.gettempdir(), fn)
do_rem = True
if cpath.isdir(filenam):
content = [
'%s\n' % s
for s in sorted(os.listdir(filenam))
]
file_data.append(content)
else:
fd = open(cpath.normpath(fn))
fdata = fd.readlines()
fd.close()
file_data.append(fdata)
# If the file was temp, it should be removed.
if do_rem:
os.remove(filenam)
modi = file_data.pop()
orig = file_data.pop()
# For snapshot views, the local directories must be removed because
# they will break the diff on the server. Just replacing
# everything before the view name (including the view name) for
# vobs do the work.
if (self.viewtype == "snapshot"
and (sys.platform.startswith('win')
or sys.platform.startswith('cygwin'))):
vinfo = self.viewinfo.rstrip("\r\n")
mnam = "c:\\\\vobs" + mnam[mnam.rfind(vinfo) + len(vinfo):]
onam = "c:\\\\vobs" + onam[onam.rfind(vinfo) + len(vinfo):]
# Call the diff lib to generate a diff.
# The dates are bogus, since they don't natter anyway.
# The only thing is that two spaces are needed to the server
# so it can identify the heades correctly.
diff = difflib.unified_diff(orig, modi, onam, mnam,
' 2002-02-21 23:30:39.942229878 -0800',
' 2002-02-21 23:30:50.442260588 -0800', lineterm=' \n')
# Transform the generator output into a string output
# Use a comprehension instead of a generator,
# so 2.3.x doesn't fail to interpret.
diffstr = ''.join([str(l) for l in diff])
# Workaround for the difflib no new line at end of file
# problem.
if not diffstr.endswith('\n'):
diffstr = diffstr + ("\n\\ No newline at end of file\n")
o.append(diffstr)
ostr = ''.join(o)
return (ostr, None) # diff, parent_diff (not supported)
raise OSError(errno.EEXIST, strerror(errno.EEXIST),
filename)
except java.io.IOException, ioe:
raise OSError(ioe)
mode = '%s%s%s%s' % (reading and 'r' or '',
(not appending and writing) and 'w' or '',
(appending and (writing or updating)) and 'a' or '',
updating and '+' or '')
if sync and (writing or updating):
from java.io import FileNotFoundException, RandomAccessFile
try:
fchannel = RandomAccessFile(sys.getPath(filename), 'rws').getChannel()
except FileNotFoundException, fnfe:
if path.isdir(filename):
raise OSError(errno.EISDIR, strerror(errno.EISDIR))
raise OSError(errno.ENOENT, strerror(errno.ENOENT), filename)
return FileIO(fchannel, mode)
return FileIO(filename, mode)
def read(fd, buffersize):
"""read(fd, buffersize) -> string
Read a file descriptor.
"""
from org.python.core.util import StringUtil
rawio = FileDescriptors.get(fd)
buf = _handle_oserror(rawio.read, buffersize)
return asPyString(StringUtil.fromBytes(buf))
def run(self):
global recording, timestamp, setpoint, pos_encodeur, pwm_envoye, erreur_derivative, erreur_integrale
while True:
timestamper = 0
while recording == True:
if serial_port.isOpen():
data = str(serial_port.readline())
list_data = data.split()
if (list_data[0] != "ORDER"):
list_data[0] = timestamper
timestamper += 50
c.writerow(list_data)
if __name__ == '__main__':
if not ntpath.isdir("../data"):
mkdir("../data")
serial_port = Serial(port="COM7", baudrate=9600)
#c = csv.writer(open("MONFICHIER.csv", "wb"))
Thread = MyThread()
Thread.start()
for p in range(50):
kp = p * 0.1 # 0.0-5.0
for i in range(40):
ki = i * 0.05 # 0.0-2.0
for d in range(60):
kd = d * 0.05 # 0.0-3.0
set_constante(kp, ki, kd)
print(kp, ki, kd)
file = open(
"../data/{kpp}_{kii}_{kdd}.csv".format(kpp=kp,
def fix_symbol_store_path(symbol_store_path = None,
remote = True,
force = False):
"""
Fix the symbol store path. Equivalent to the C{.symfix} command in
Microsoft WinDbg.
If the symbol store path environment variable hasn't been set, this
method will provide a default one.
@type symbol_store_path: str or None
@param symbol_store_path: (Optional) Symbol store path to set.
@type remote: bool
@param remote: (Optional) Defines the symbol store path to set when the
C{symbol_store_path} is C{None}.
If C{True} the default symbol store path is set to the Microsoft
symbol server. Debug symbols will be downloaded through HTTP.
This gives the best results but is also quite slow.
If C{False} the default symbol store path is set to the local
cache only. This prevents debug symbols from being downloaded and
is faster, but unless you've installed the debug symbols on this
machine or downloaded them in a previous debugging session, some
symbols may be missing.
If the C{symbol_store_path} argument is not C{None}, this argument
is ignored entirely.
@type force: bool
@param force: (Optional) If C{True} the new symbol store path is set
always. If C{False} the new symbol store path is only set if
missing.
This allows you to call this method preventively to ensure the
symbol server is always set up correctly when running your script,
but without messing up whatever configuration the user has.
Example::
from winappdbg import Debug, System
def simple_debugger( argv ):
# Instance a Debug object
debug = Debug( MyEventHandler() )
try:
# Make sure the remote symbol store is set
System.fix_symbol_store_path(remote = True,
force = False)
# Start a new process for debugging
debug.execv( argv )
# Wait for the debugee to finish
debug.loop()
# Stop the debugger
finally:
debug.stop()
@rtype: str or None
@return: The previously set symbol store path if any,
otherwise returns C{None}.
"""
try:
if symbol_store_path is None:
local_path = "C:\\SYMBOLS"
if not ntpath.isdir(local_path):
local_path = "C:\\Windows\\Symbols"
if not ntpath.isdir(local_path):
local_path = ntpath.abspath(".")
if remote:
symbol_store_path = (
"cache*;SRV*"
+ local_path +
"*"
"http://msdl.microsoft.com/download/symbols"
)
else:
symbol_store_path = "cache*;SRV*" + local_path
previous = os.environ.get("_NT_SYMBOL_PATH", None)
if not previous or force:
os.environ["_NT_SYMBOL_PATH"] = symbol_store_path
return previous
except Exception, e:
warnings.warn("Cannot fix symbol path, reason: %s" % str(e),
RuntimeWarning)
allFiles.append(fullPath)
return allFiles
SuperPath = input("[*] Veuillez saisir le chemin du tri : ")
print("Chargement ...")
# SuperPath = "./TestFolder/"
# if SuperPath[-1] == "/" or SuperPath[-1] == "\\":
# SuperPath
# pass
# FilesList = list()
# SuperPath = SuperPath.rstrip('/')
# SuperPath = SuperPath.rstrip('\\')
# for target in getListOfFiles(SuperPath):
# if ntpath.basename(target) != "desktop.ini":
# os.rename(target, SuperPath + "/" + ntpath.basename(target))
# FilesList.append(SuperPath + "/" + ntpath.basename(target))
# else:
# os.remove(target)
# for File in os.listdir(SuperPath):
# if os.path.isdir(os.path.join(SuperPath, File)):
# os.removedirs(os.path.join(SuperPath, File))
for File in os.listdir(SuperPath):
FilePath = os.path.join(SuperPath, File)
if ntpath.isdir(FilePath) != True:
Date = os.path.getmtime(FilePath)
FolderName = str(datetime.datetime.fromtimestamp(Date).year) + "-" + str(datetime.datetime.fromtimestamp(Date).month).zfill(2)
FolderPath = os.path.join(SuperPath, FolderName)
Path(FolderPath).mkdir(parents=True, exist_ok=True)
os.rename(FilePath, os.path.join(SuperPath, FolderName, File))
