def get_ops(orig_seq, modified_seq, in_dir, out_dir): """ turn to and from sequence objects in to list of filename level operations """ ops_list = [] for pair in zip(list_files(orig_seq), list_files(modified_seq)): # list filesequences out as original modified -> file pairs ops_list.append((get_dir_file(in_dir, pair[0]), get_dir_file(out_dir, pair[1]))) active_ops = filter(lambda pair: pair[0] != pair[1], ops_list) # Filter out those filepairs where to/from are identical return active_ops
def download_all(service_stub, file_server, destination_dir=os.path.expanduser("~/Downloads")):
files = list_files.list_files(service_stub, quiet=True)
for f in files:
fname = os.path.basename(f)
url = "http://" + file_server + "/" + fname
print("Downloading %s" % url)
urllib.request.urlretrieve(url, destination_dir + "/" + fname)
def fscmd(original_seqlist, modified_seqlist, cmd, execute, quiet):
args_proto = cmd[1].split(" ")
for pair in zip(original_seqlist, modified_seqlist):
originals = list_files(pair[0]) if pair[0] != None else list_files(pair[1])
modifieds = list_files(pair[1])
for entry in zip(originals, modifieds):
args = map(lambda x: x.replace("%original", entry[0]), args_proto)
args = map(lambda x: x.replace("%modified", entry[1]), args)
if execute:
subprocess.check_call([cmd[0]] + args)
else:
print_cli("dry-run: " + cmd[0] + " " + " ".join(args) + "\n", quiet)
if not execute:
print_cli("run with --execute to actually run these operations\n", quiet)
def testNoFiles(self, m_isfile, m_listdir, m_getcwd):
"""Can handle lack of files to list."""
# Setup
m_getcwd.return_value = 'test'
m_listdir.return_value = []
m_isfile.return_value = True
expected = []
# Run
result = list_files.list_files()
assert expected == result
def testNotFile(self, m_isfile, m_listdir, m_getcwd):
"""Should not list files that are not files."""
# Setup
m_getcwd.return_value = 'test'
m_listdir.return_value = ['one', 'two']
m_isfile.side_effect = [True, False]
expected = ['one']
# Run
result = list_files.list_files()
assert expected == result
def testSimple(self, m_isfile, m_listdir, m_getcwd):
"""Should be able to list files."""
# Setup
m_getcwd.return_value = 'test'
m_listdir.return_value = ['one.txt', 'two', 'three.py']
m_isfile.return_value = True
expected = ['one.txt', 'two', 'three.py']
# Run
result = list_files.list_files()
assert expected == result
def fill_counters():
fpaths = list_files("../data/")
# fpaths = ['../data/pg1342.txt']
unigrams = Counter()
bigrams = Counter()
for i, fn in enumerate(fpaths[:1]):
if i > 1 and i % 50 == 0:
print i
with open(fn, "r") as f:
doc = f.read().decode("utf8")
sentences = nltk.sent_tokenize(doc)
tokenized_sentences = [nltk.word_tokenize(s) for s in sentences]
tagged_sentences = [nltk.pos_tag(s) for s in tokenized_sentences]
chunked_sentences = [nltk.ne_chunk(s, binary=True) for s in tagged_sentences]
for csent in chunked_sentences:
words = [(n[0], isinstance(n[0], tuple)) for n in csent]
# add unigrams
for word in words:
if not word[1]:
w = word[0].lower()
unigrams.update([w])
for pair in zip(words, words[1:]):
if not pair[0][1] and not pair[1][1]:
t = (pair[0][0].lower(), pair[1][0].lower())
bigrams.update([t])
# # normalize counters
for key in bigrams:
if bigrams[key] < 2: # remove rare occurrences (was 5 before)
bigrams[key] = 0
# print unigrams
normalize_counter(unigrams)
normalize_counter(bigrams)
return (unigrams, bigrams)
def enf_filelist(filelist, extension=None):
"""
Sanitizes file list inputs
This function checks that the input is a list of files and not a directory. If the input
is a directory, then it returns a list of files in the directory which match the desired
extension. This is to allow all functions which input filelists to be a little more
flexible by accepting directories instead.
:param filelist: a list of filepath strings
:param extension: output list contains only files with this string extension. (txt, tif, etc)
:return new_filelist: sanitized file list
"""
new_filelist = None
if isinstance(filelist, str):
if os.path.isdir(filelist):
new_filelist = list_files(False, filelist, False, False)
elif os.path.isfile(filelist):
new_filelist = [filelist]
elif isinstance(filelist, list):
new_filelist = filelist
elif isinstance(filelist, bool) or isinstance(filelist, None):
raise TypeError(
'Expected file list or directory but received boolean or None type input!'
)
if new_filelist is None:
new_filelist = filelist
if extension is not None:
new_filelist = [
new_file for new_file in new_filelist if extension in new_filelist
]
return new_filelist
def enf_filelist(filelist, extension = None):
"""
Sanitizes file list inputs
This function checks that the input is a list of files and not a directory. If the input
is a directory, then it returns a list of files in the directory which match the desired
extension. This is to allow all functions which input filelists to be more flexible by
accepting directories instead.
"""
new_filelist = None
if isinstance(filelist, str):
if os.path.isdir(filelist):
new_filelist = list_files(False, filelist, False, False)
elif os.path.isfile(filelist):
new_filelist = [filelist]
elif isinstance(filelist, bool):
print 'Expected file list or directory but received boolean or None type input!'
return False
elif isinstance(filelist, list):
new_filelist = filelist
if new_filelist is None:
new_filelist = filelist
if extension is not None:
for new_file in new_filelist:
if extension not in new_file:
new_filelist.remove(new_file)
return new_filelist
def enf_filelist(filelist, extension = None):
"""
Sanitizes file list inputs
This function checks that the input is a list of files and not a directory. If the input
is a directory, then it returns a list of files in the directory which match the desired
extension. This is to allow all functions which input filelists to be a little more
flexible by accepting directories instead.
:param filelist: a list of filepath strings
:param extension: output list contains only files with this string extension. (txt, tif, etc)
:return new_filelist: sanitized file list
"""
new_filelist = None
if isinstance(filelist, str):
if os.path.isdir(filelist):
new_filelist = list_files(False, filelist, False, False)
elif os.path.isfile(filelist):
new_filelist = [filelist]
elif isinstance(filelist, list):
new_filelist = filelist
elif isinstance(filelist, bool) or isinstance(filelist, None):
raise TypeError('Expected file list or directory but received boolean or None type input!')
if new_filelist is None:
new_filelist = filelist
if extension is not None:
new_filelist = [new_file for new_file in new_filelist if extension in new_filelist]
return new_filelist
def upload_files(drive, folder_id, files, silent):
"Print file information into a file"
import sys
ls = list_files(drive, folder_id)
for fname in files:
if fname in ls:
if not silent:
sys.stdout.write('Replacing file ' + fname + ' ... ')
f = drive.CreateFile({'id': ls[fname][0]})
else:
f = drive.CreateFile({"parents":
[{"kind": "drive#fileLink",
"id": folder_id}]})
if not silent:
sys.stdout.write('Uploading file ' + fname + ' ... ')
f.SetContentFile(fname)
f.Upload()
if not silent:
sys.stdout.write('Done\n')
def enf_filelist(filelist, extension=None):
"""
Sanitizes file list inputs
This function checks that the input is a list of files and not a directory. If the input
is a directory, then it returns a list of files in the directory which match the desired
extension. This is to allow all functions which input filelists to be more flexible by
accepting directories instead.
"""
new_filelist = None
if isinstance(filelist, str):
if os.path.isdir(filelist):
new_filelist = list_files(False, filelist, False, False)
elif os.path.isfile(filelist):
new_filelist = [filelist]
elif isinstance(filelist, bool):
print 'Expected file list or directory but received boolean or None type input!'
return False
elif isinstance(filelist, list):
new_filelist = filelist
if new_filelist is None:
new_filelist = filelist
if extension is not None:
for new_file in new_filelist:
if extension not in new_file:
new_filelist.remove(new_file)
return new_filelist
#!/usr/bin/python
# -*- coding: utf8 -*-
# -*-coding:cp1251 -*-
# ============================================================
#
#
# ============================================================
import os
import urllib2
import time
import zlib
from list_files import list_files
# main entrance point:
if __name__ == "__main__":
print u"Main app start."
print ""
search_path_main_box = u"d:/dev"
# search_path_main_box = u"d:/TMP"
# search_path_main_box = u"d:/Documents and Settings/aa027762/My Documents/[0] INBOX"
_lst_fls = list_files(search_path_main_box)
_lst_fls.run(True)
print ""
print "Main program end."
except requests.exceptions.RequestException as cerror:
print("Error processing request", cerror)
if response.json():
print(json.dumps(response.json(), indent=2))
sys.exit(1)
taskid = response.json()['response']['taskId']
print("Waiting for Task %s" % taskid)
task_result = wait_on_task(taskid, token)
return task_result
if __name__ == "__main__":
if len(sys.argv) >1 and sys.argv[1] == "-a":
filelist = list_files('config')['response']
filetuple = [(file['name'], file['id']) for file in filelist]
project_name_list = set()
f = open(DEVICES, 'rt')
try:
reader = csv.DictReader(f)
for dict_row in reader:
filename = name_wrap(dict_row['hostName'] + "-config")
fileid = [ id for fn, id in filetuple if fn == filename][0]
response = create_project_rule(name_wrap(dict_row['site']),
serial=name_wrap(dict_row['serialNumber'], fixed_len=True),
platform=dict_row['platformId'],
host=dict_row['hostName'],
config_file_id=fileid)
def uploaded_files():
return render_template('files.html',file_list=list_files.list_files(UPLOAD_FOLDER))
# Athenticate
src_dir = os.path.dirname(os.path.realpath(__file__))
gauth = authenticate(src_dir, "r")
# Create drive object
drive = GoogleDrive(gauth)
# Obtain file name and size
if args.id:
file_id = args.id
f = drive.CreateFile({'id': args.id})
size = int(f['fileSize'])
filename = f['title']
elif args.file:
ls = list_files(drive, args.parent)
try:
file_id, size = ls[args.file]
except:
print('File', args.file, 'does not exist.')
sys.exit(-1)
filename = args.file
if args.remote_name:
outfile = filename
else:
outfile = args.outfile
# Disable progressbar in silient mode
if args.silent:
size = 0
def main():
file_convert = list_files()
image_conversion(file_convert)
print("Markdown links converted")
nmseitol = 0.75
# would be better to have the program check this automatically
grid_coord_check = 'LL'
data_coord_check = 'LL'
# allow users to provide a grid of their own but someow specify what it
# needsto look like
grid_filename = ' '
#################### end inputs ###########################################
# Call dataBuilder to construct data in necessary format for interpolation
os.chdir(toolkitpath)
from list_files import list_files
filelist = list_files(datapath, datatype)
from dataBuilder import dataBuilder
os.chdir(datapath)
t = time.time()
x, z = dataBuilder(filelist, data_coord_check)
s = np.ones((np.size(x[:, 1]), 1))
lfile = np.shape(filelist)
elapsed = time.time() - t
print 'data building time is %d seconds' % elapsed
# Call grid builder to make a grid based on x,y min and max values
os.chdir(toolkitpath)
from gridBuilder import gridBuilder
x_grid, y_grid = gridBuilder(x0, x1, y0, y1, dx, dy, grid_coord_check,
grid_filename)
url = create_url(path="file/%s" % fileid)
print("DELETE %s" % url)
headers= { 'x-auth-token': token['token']}
try:
response = requests.delete(url, headers=headers, verify=False)
except requests.exceptions.RequestException as cerror:
print("Error processing request", cerror)
sys.exit(1)
return response.json()
if __name__ == "__main__":
if len(sys.argv) >1 and sys.argv[1] == "-a":
filelist = list_files('config')['response']
filetuple = [(file['name'], file['id']) for file in filelist]
for filename in os.listdir(CONFIGS_DIR):
if not filename.startswith('.'):
print("deleting %s" %filename)
try:
fileid = [ id for fn, id in filetuple if fn == filename][0]
except IndexError as e:
print("file %s not present" % filename)
else:
response = delete_file("config", fileid)
print(json.dumps(response['response'], indent=2))
else:
response = delete_file("config", sys.argv[1])
print(json.dumps(response['response'], indent=2))
def do_list_files(self, input):
'''list_files
Lists the supported media files in the data directory of the server
'''
list_files.list_files(self._service)
