class MyTestCase(unittest.TestCase):
def setUp(self):
self.fm = FileManager()
def test_fileManager_invalidDirectoryRaisesError(self):
with self.assertRaisesRegex(file_manager.InvalidDirectoryError,
" is not a valid directory"):
fm = FileManager(local_download_folder="lalala/download")
@mock.patch.object(file_manager.os.path, 'isdir')
@mock.patch.object(file_manager.os, 'rename')
def test_fileManager_moveFilesToExternalStorage_checksIfStorageExists(
self, mock_rename, mock_isdir):
mock_isdir.return_value = True
self.fm.move_file_to_external_storage("haha")
mock_isdir.assert_called() # Check if storage directory exists
mock_rename.assert_called() # Check if file was moved
def test_moveFilesToExternalStorage_raisesErrorIfStorageNotFound(self):
with self.assertRaisesRegex(
file_manager.ExternalStorageNotFound,
"not found. Ensure your storage is mounted properly and that "
"the directory is valid."):
self.fm.move_file_to_external_storage("")
@mock.patch.object(file_manager.os, 'listdir')
def test_listFiles_getsListOfPaths(self, mock_listdir):
mock_listdir.return_value = []
self.assertEqual([], self.fm.list_local_download_files(),
"List local files should use listdir.")
def tick(self, up, down, enter, esc):
if esc:
exit(0)
if up and self.blocker:
self.blocker = False
if self.selected_index == 0:
self.selected_index = len(self.menu_text) - 1
else:
self.selected_index -= 1
elif not up and not self.blocker and not down and not enter:
self.blocker = True
if down and self.blocker:
self.blocker = False
if self.selected_index == len(self.menu_text) - 1:
self.selected_index = 0
else:
self.selected_index += 1
elif not down and not self.blocker and not up and not enter:
self.blocker = True
if enter and self.blocker:
if self.selected_index == 0:
self.set_game_scene = True
elif self.selected_index == 1:
FileManager.save('objects.json')
elif self.selected_index == 2:
pygame.quit()
exit(0)
self.blocker = False
elif not down and not self.blocker and not up and not enter:
self.blocker = True
def do_series_metadata(input_file_path, config, debug):
try:
thetvdb = TheTvDb(config, debug)
moviedb = MovieDB(config, debug)
database = Database(config, debug)
file_manager = FileManager(config, database, thetvdb, moviedb, debug)
match = file_manager.match_file(input_file_path, True)
if match is None:
print "File name '%s' doesn't match any series on the watch list." % input_file_path,
return 2
else:
(file_name, series, episode) = match
metadata = file_manager.generate_episode_metadata(episode)
if metadata is None:
print "Error generating metadata for file '%s'." % (file_name, )
return 4
for l in metadata:
print l
return 0
except:
traceback.print_exc()
return 11
def __init__(self, projectPath='', projectName='', mainPath=''):
self.projectPath = projectPath
self.projectName = projectName
self.mainPath = mainPath
self.javaManager = JavaManager()
self.zipManager = ZipManager()
self.fileManager = FileManager()
def merge_cities_file(cls, _format="csv"):
"""
Transform the file for the RH
:param _format: two possibilities:
- "csv": merge the extracted cities into a csv file
- "json": merge the extracted cities into a json file
:return:
"""
data = {} if _format == "json" else []
for place_type in cls.PLACE_TYPES:
folder = os.path.join(cls.CITIES_FOLDER_NAME, place_type)
for filename in FileManager.list_files(folder):
cities = FileManager.read(os.path.join(folder, filename),
_format="json")
if cities['status'] == 'SUCCESS':
cities = cities['nodes']
if _format == "json":
data.update(cls.to_json(cities))
elif _format == "csv":
data = cls.to_csv(data, cities)
if _format == "csv": # we complete the line with None. We don't have the name ine each language
l = len(data[0])
for line in data[1:]:
for i in range(len(line), l):
line.append(None)
FileManager.write(os.path.join(
cls.CITIES_FOLDER_NAME,
cls.WORLD_CITIES_FILE.format(format=_format)),
data,
_format=_format)
def story():
global translator
headers = request.headers
title = request.args.get('title')
if valid_api_key(headers):
my_story = StoryManager().fetch_story(story=title)
story_title = my_story[0]
new_story = Story(title=story_title)
db.session.add(new_story)
db.session.commit()
story_id = new_story.id
story_paragraphs = my_story[1]
if not translator:
translator = SeleniumTranslationManger()
for paragraph in story_paragraphs:
new_paragraph = Paragraph(es=paragraph,
en=translator.translate(
text=paragraph,
title=story_title),
story_id=story_id)
db.session.add(new_paragraph)
print("translating paragraph")
db.session.commit()
translator.close_webdriver()
file_manager = FileManager()
print(file_manager.return_story(story_title))
return_value = {"story-title": f"{story_title}"}
return jsonify(response=return_value)
else:
return "API Key not found", 403
def __train_metis_model(self, c):
matrices = FileManager().get_ensembling_matrices(
self.name, self.are_clusters_fixed)
for key, value in matrices.items():
if key == 'CO':
model = MetisClustering(value, c)
assignations = model.clusterize()[1]
FileManager().save_results(assignations, self.name, key,
self.are_clusters_fixed, 'Metis')
else:
features = len(value[0])
graph = nx.Graph()
graph.add_nodes_from(list(range(self.num_rows)))
graph.add_nodes_from(
list(range(self.num_rows, self.num_rows + features)))
for cnt_row, rows in enumerate(value):
for cnt_col, col in enumerate(rows):
graph.add_edge(cnt_row,
self.num_rows + cnt_col,
weight=col)
model = MetisClustering(value, c, graph)
assignations = model.clusterize()[1][:self.num_rows]
FileManager().save_results(assignations, self.name, key,
self.are_clusters_fixed, 'Metis')
Validator(self.name, self.numerical, self.ground_truth, 'Metis',
self.are_clusters_fixed).validate_consensus()
def run_single_clusterings(self):
self.preprocess()
is_fixed = 'fixed_k' if self.are_clusters_fixed else 'random_k'
if len(
os.listdir(
os.path.join('clusters', self.name, is_fixed,
'Kmeans'))) != 10:
for i in range(Experiment.ENSEMBLING_EXPERIMENTS):
print(
"Single crispy clustering: \033[1m{}\033[0m (N={}) exp={}\n"
.format(self.name, self.num_rows, i))
FileManager().save_csv(self.numerical, self.name)
_, gt_indices = np.unique(self.ground_truth,
return_inverse=True)
c = self.num_fixed_c if self.are_clusters_fixed else self.num_random_c
self.__train_kmeans_ensemblers(c, i)
print()
if len(
os.listdir(
os.path.join('clusters', self.name, is_fixed,
'FuzzyMeans'))) != 10:
for i in range(Experiment.ENSEMBLING_EXPERIMENTS):
print(
"Single fuzzy clustering: \033[1m{}\033[0m (N={}) exp={}\n"
.format(self.name, self.num_rows, i))
FileManager().save_csv(self.numerical, self.name)
_, gt_indices = np.unique(self.ground_truth,
return_inverse=True)
c = self.num_fixed_c if self.are_clusters_fixed else self.num_random_c
self.__train_fuzzy_c_means_ensemblers(c, i)
print()
def main():
print("Read embeddings")
path = 'data/'
embeddings_in = '{}funfams_L1_tucker128.npy'.format(path)
embedding_ids_in = '{}funfams_ids.txt'.format(path)
embeddings = Npy2Npz.get_dataset_uncompressed(embeddings_in,
embedding_ids_in)
print("Read FunFam info")
funfam_info_in = '{}cath_v4_3-funfams-summary.tsv'.format(path)
funfam_info = FileManager.read_funfam_ids_with_family(funfam_info_in)
print("Read sequence information")
sequence_info_in = 'data/statistics_funfams_v4.3.txt'
sequences = FileManager.read_sequence_info(sequence_info_in,
embeddings.keys())
fm = FileManager(ungapped_aln_path='funfam_data')
dist_calculator = EmbeddingDistance(funfam_info, embeddings, sequences, fm)
superfamilies_in = '{}funfam_binding_stats_test.txt'.format(path)
superfamilies = FileManager.read_funfam_ids_with_family(
superfamilies_in, 0, 1)
print("Calculate distances")
for s in superfamilies.keys():
funfams = funfam_info[s]
for f in funfams:
if len(sequences[f]) > 1:
print('{}\t{}'.format(s, f))
dist_calculator.calc_distances_within_funfam(f, False)
def main():
file_manager = FileManager()
file_manager.create_download_directory()
last_run_date = file_manager.get_most_recent_download_date()
if last_run_date is None:
start_date = constants.EARLIEST_GRANT_AWARD_DATE
else:
start_date = last_run_date + datetime.timedelta(days=1)
if start_date <= constants.MOST_RECENT_GRANT_AWARD_DATE:
download_period = DownloadPeriod(start_date)
while download_period:
# This is really inefficient, but we're going to create a new browser session for each request. This is
# because of how report results are presented. It would be difficult to determine which results go with which
# date ranges otherwise.
with Scraper() as scraper:
if scraper.get_query_results(file_manager, download_period):
download_period = download_period.increment_period()
else:
download_period.downgrade_period()
else:
print "Downloads directory is up to date. Quitting."
def main():
fm = FileManager()
fe = Fetcher()
folder_dir = fm.get_folder_dir()
df = [fe.statistics(), fe.coutries_data(), fe.countries_historical_data()]
filenames = [
'statistics', 'all_country_data', 'all_country_historical_data'
]
for i in range(3):
filename = '{}/{}.csv'.format(folder_dir, filenames[i])
df[i].to_csv(filename, index=False)
print('{} is created successfully.'.format(filename))
if len(argv) > 1:
country = argv[1]
df = [fe.coutry_data(country), fe.country_historical_data(country)]
filenames = [
'{}__country_data'.format(country),
'{}__country_historical_data'.format(country)
]
for i in range(2):
filename = '{}/{}.csv'.format(folder_dir, filenames[i])
df[i].to_csv(filename, index=False)
print('{} is created successfully.'.format(filename))
def do_movie_metadata(input_file_path, config, debug):
try:
moviedb = MovieDB(config, debug)
database = Database(config, debug)
thetvdb = TheTvDb(config, debug)
file_manager = FileManager(config, database, thetvdb, moviedb, debug)
match = file_manager.match_movie_file(input_file_path, True)
if match is None:
print "File name '%s' doesn't match any movie in the local cache or on IMDb." % input_file_path,
return 2
else:
(file_name, movie, disc) = match
metadata = file_manager.generate_movie_metadata(movie)
if metadata is None:
print "Error generating metadata for file '%s'." % (file_name, )
return 4
for l in metadata:
print l
return 0
except:
traceback.print_exc()
return 11
def do_build_genres(input_directory, genre_path, is_series_dir, act_immediately, config, debug):
if not os.path.exists(input_directory):
print "Directory '%s' does not exist. Exiting." % input_directory,
return 1
try:
thetvdb = TheTvDb(config, debug)
moviedb = MovieDB(config, debug)
database = Database(config, debug)
file_manager = FileManager(config, database, thetvdb, moviedb, debug)
files = []
for root, dir, files_to_add in os.walk(input_directory):
for file in files_to_add:
if file_manager.is_media_file(file):
files.append((root, file))
for (root, file) in files:
if is_series_dir:
build_genres_for_episode_file(file_manager, genre_path, root, file, act_immediately)
else:
build_genres_for_movie_file(file_manager, genre_path, root, file, act_immediately)
return 0
except:
traceback.print_exc()
return 11
def __init__(self, connection=None, address=None, password_hash=None):
super().__init__()
self.connection = connection
self.address = address
self.data_received = ''
self.function_switcher = self.setup_function_switcher()
self.file_manager = FileManager()
self.simplified_abs_path = ''
self.item_name = ''
self.state = USER_AUTH
self.password_hash = password_hash
if connection is None:
self.address = ('0.0.0.0', int(input('Which port to open the server? ')))
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind(self.address)
self.server.listen(5)
self.password_hash = sha256(input('What will be the server password? ').encode()).hexdigest()
self.execute_server_listener()
else:
self.server_loop()
def merge_cities_file(cls, _format="csv"):
res = []
codes = {
country: code
for code, country in bc.Locale("en", "US").territories.iteritems()
}
for country in cls.request_country_codes().iterkeys():
cities = defaultdict(dict)
if country == "United States of America":
short_country = "United States"
elif country == "People's Republic of China":
short_country = "China"
else:
short_country = country
country_code = codes.get(short_country, "ZZ")
for lang in cls.SUPPORTED_LANGUAGES:
filename = cls.get_file_name(country, lang)
if FileManager.exists(filename):
for element in FileManager.read(filename, _format="json"):
cities[element["city"]].update(element)
cities[element["city"]]['name:%s' %
lang] = element["cityLabel"]
if _format == "json":
res = cls.to_json(res, cities.itervalues(), country_code)
elif _format == "csv":
res = cls.to_csv(res, cities.itervalues(), country_code)
else:
raise TypeError("Format %s not supported" % _format)
FileManager.write(os.path.join(
cls.CITIES_FOLDER_NAME,
cls.WORLD_CITIES_FILE.format(format=_format)),
res,
_format=_format)
def rename_files(self, event):
# self.folder_path = "/Volumes/Backup Drive/Movies"
if self.folder_path is None:
print("Please chose a folder to rename files")
return
FileManager.rename_files_by_replacing_string(self.folder_path, self.files_in_folder,
"www.TamilRockers.ws - ", "***###***")
print("Rename files pressed")
def upload_new_model():
f = request.files['screenshot']
file_manager = FileManager(os.getenv('S3_MODEL_BUCKET'),
{'application/octet-stream'})
file = file_manager.upload_new_version(f, 'greenlight_model.pkl')
return jsonify(file)
def __init__(self, server_instance, clientsocket, addr):
self.server_ip = addr[0]
self.client_id = addr[1] # important
self.server = server_instance # needed to use & alter <clients, names, rooms>
self.clientsocket = clientsocket # needed for sending and receiving
self.torrent = Torrent("age.torrent")
self.file_manager = FileManager(peer_id=0, torrent=self.torrent)
def __init__(self) -> None:
super().__init__()
self.file_manager = FileManager()
self.scheduled_deliveries_hour = None
self.handler_running = False
self.scheduler_running = False
self.logger = log_manager.get_logger('session_manager')
def test2_converttest(self):
file1 = FileManager('test_files/representation.pdf')
file2 = FileManager('test_files/representation.pdf', '.')
convert([file1, file2], ['txt'])
self.failUnless(open('test_files/representation.txt'))
self.failUnless(open('representation.txt'))
def test_write_file_data(self):
file_manager = FileManager()
finder = ItemFinder()
handle = finder.find("/")
user_handle = finder.find("/users/system")
file_version, file_length, file_hash = file_manager.write_file_data(handle.item_id, None, b'0000000000', user_handle)
self.assertEquals(0, file_version)
self.assertEquals(10, file_length)
self.assertEquals("30e26d585a255e74b26f46809f0b67d149443f3e372d1536afc21864431270e4", file_hash)
def test2_htmlpdf2(self):
file1 = FileManager('test_files/test1.html', '.')
file2 = FileManager('test_files/test2.html', '.')
file3 = FileManager('test_files/test3.html', '.')
test = HtmlPdf([file1, file2, file3])
test.convert()
self.failUnless(open('test1.pdf'))
self.failUnless(open('test2.pdf'))
self.failUnless(open('test3.pdf'))
def test_write_file_data_large(self):
file_manager = FileManager()
finder = ItemFinder()
handle = finder.find("/")
user_handle = finder.find("/users/system")
data = b'0' * 13000000
file_version, file_length, file_hash = file_manager.write_file_data(handle.item_id, None, data, user_handle)
self.assertEquals(0, file_version)
self.assertEquals(13000000, file_length)
self.assertEquals("5c03fc7510021b52b0aacf899290a32e5c19218bf693a509fa4039a813101644", file_hash)
def test_cleanup(self):
test_url = "https://www.google.pl" # szybsze
test_manager = FileManager(test_url)
test_manager.save_to_zip(download_text=True, download_images=True)
now = datetime.now()
domain = tldextract.extract(test_url).domain
file_path = f"{domain}--{now.year}_{now.month}_{now.day}--{now.hour}_{now.minute}--images_text.zip"
self.assertTrue(not os.path.exists(test_manager.tmp_download_dir))
os.remove("logs.txt")
os.remove(file_path)
def test1_pdfhtml1(self):
file1 = FileManager('test_files/test1.pdf')
file2 = FileManager('test_files/test2.pdf')
file3 = FileManager('test_files/test3.pdf')
test = PdfHtml([file1, file2, file3])
test.convert()
self.failUnless(open('test_files/test1.html'))
self.failUnless(open('test_files/test2.html'))
self.failUnless(open('test_files/test3.html'))
def __train_spectral_model(self, c):
matrices = FileManager().get_ensembling_matrices(
self.name, self.are_clusters_fixed)
for key, value in matrices.items():
model = Spectral(value, c)
assignations = model.clusterize()
FileManager().save_results(assignations, self.name, key,
self.are_clusters_fixed, 'Spectral')
Validator(self.name, self.numerical, self.ground_truth, 'Spectral',
self.are_clusters_fixed).validate_consensus()
def __train_ahc_model(self, c, link=None, algorithm='SL'):
matrices = FileManager().get_ensembling_matrices(
self.name, self.are_clusters_fixed)
for key, value in matrices.items():
model = Agglomerative(value, c, ahc_linkage=link)
assignations = model.clusterize()
FileManager().save_results(assignations, self.name, key,
self.are_clusters_fixed, algorithm)
Validator(self.name, self.numerical, self.ground_truth, algorithm,
self.are_clusters_fixed).validate_consensus()
def __init__(self):
self.SAY_SPEED = 95
self.final_question = False
self.random_question = None
self.answer_1 = None
self.answer_2 = None
self.answer_3 = None
self.correct_answer_index = None
self.correct_answer = None
self.file_manager = FileManager()
self.__instructions()
def test1_converttest(self):
file1 = FileManager('test_files/test1.html', )
file2 = FileManager('test_files/test2.html', )
file3 = FileManager('test_files/test3.html', '.')
file4 = FileManager('test_files/test4.html', '.')
convert([file1, file2, file3, file4], ['txt'])
self.failUnless(open('test_files/test1.txt'))
self.failUnless(open('test_files/test2.txt'))
self.failUnless(open('test3.txt'))
self.failUnless(open('test4.txt'))
def main():
hyper_params = {
"learning_rate": 1,
"style_weight": 10000,
"content_weight": 1,
"n_steps": 300
}
experiment = Experiment(api_key="a604AfX0S9Bmt6HdpMHxg9MCI",
project_name="style-transfer",
workspace="polmonroig")
experiment.log_parameters(hyper_params)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if torch.cuda.is_available():
print("Currently using cuda device.")
# define file paths and directories
content_images_dir = "content_images/"
style_images_dir = "style_images/"
output_images_dir = "output_images/"
content_image_name = "content_01.png"
style_image_name = "style_01.jpg"
output_image_path = join(
output_images_dir,
content_image_name.split('.')[0] + "_" +
style_image_name.split('.')[0] + ".jpg")
content_image_path = join(content_images_dir, content_image_name)
style_image_path = join(style_images_dir, style_image_name)
# define image file manager
max_shape = (800, 800)
fileManager = FileManager(content_image_path, style_image_path, device,
max_shape)
# read images
content_image, style_image = fileManager.read_images()
input_image = content_image.clone()
model = ArtNet(device=device)
output_image = model.train(hyper_params['content_weight'],
hyper_params['style_weight'],
hyper_params['n_steps'], content_image,
style_image, input_image,
hyper_params['learning_rate'], experiment)
fileManager.save_image(output_image, output_image_path)
experiment.log_image(
output_image_path,
content_image_name.split('.')[0] + "_" +
style_image_name.split('.')[0])
def test_write_file_data(self):
file_manager = FileManager()
finder = ItemFinder()
handle = finder.find("/")
user_handle = finder.find("/users/system")
file_version, file_length, file_hash = file_manager.write_file_data(
handle.item_id, None, b'0000000000', user_handle)
self.assertEquals(0, file_version)
self.assertEquals(10, file_length)
self.assertEquals(
"30e26d585a255e74b26f46809f0b67d149443f3e372d1536afc21864431270e4",
file_hash)
def fileHandler(promt, parameter):
if (promt == 'date'):
date_str = ' '
date_str = date_str.join(parameter)
print(date_str)
try:
new_date = datetime.strptime(date_str, '%Y %m %d')
manager = FileManager(new_date)
manager.run()
except ValueError as e:
print(e)
pass
def test2_htmltxt2(self):
file1 = FileManager('test_files/test1.html', '.')
file2 = FileManager('test_files/test2.html', '.')
file3 = FileManager('test_files/test3.html', '.')
file4 = FileManager('test_files/test4.html', '.')
test = HtmlTxt([file1, file2, file3, file4])
test.convert()
self.failUnless(open('test1.txt'))
self.failUnless(open('test2.txt'))
self.failUnless(open('test3.txt'))
self.failUnless(open('test4.txt'))
def generate_data(self, filename):
print "# # Generating data..."
for datum in self.data['objects']:
print '-'*40
print " "*10,datum["name"]
print '-'*40
print "- Addresse: ",datum["street_address"]," CP:",datum["postal_code"]
print "- Phone : (+33) ",datum["phone"]
print "# # Writing data on : '",filename, "' ..."
f = FileManager(filename, 'w')
f.make_file(self.data)
def test2_txthtml1(self):
file1 = FileManager('test_files/test1.txt')
file2 = FileManager('test_files/test2.txt')
file3 = FileManager('test_files/test3.txt')
file4 = FileManager('test_files/test4.txt')
test = TxtHtml([file1, file2, file3, file4])
test.convert()
self.failUnless(open('test_files/test1.html'))
self.failUnless(open('test_files/test2.html'))
self.failUnless(open('test_files/test3.html'))
self.failUnless(open('test_files/test4.html'))
def test_write_file_data_large(self):
file_manager = FileManager()
finder = ItemFinder()
handle = finder.find("/")
user_handle = finder.find("/users/system")
data = b'0' * 13000000
file_version, file_length, file_hash = file_manager.write_file_data(
handle.item_id, None, data, user_handle)
self.assertEquals(0, file_version)
self.assertEquals(13000000, file_length)
self.assertEquals(
"5c03fc7510021b52b0aacf899290a32e5c19218bf693a509fa4039a813101644",
file_hash)
def __init__(self):
super().__init__()
self.address = ('', 0)
self.state = USER_AUTH
self.switch_function = self.initiate_switch_function()
self.current_path = '/'
self.send_file_path = ''
self.file_manager = FileManager(abs_root_folder=True)
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.client_loop()
self.socket.close()
def test_file_name_generation(self):
file_manager = FileManager()
self.assertRaises(ValueError, file_manager.generate_full_file_path, None, None)
self.assertRaises(ValueError, file_manager.generate_full_file_path, date(2001, 1, 1), None)
self.assertRaises(ValueError, file_manager.generate_full_file_path, None, date(2001, 1, 1))
self.assertRaises(TypeError, file_manager.generate_full_file_path, "2001-01-01", date(2001, 1, 1))
self.assertRaises(TypeError, file_manager.generate_full_file_path, date(2001, 1, 1), "2001-01-01")
self.assertRaises(ValueError, file_manager.generate_full_file_path, date(2005, 1, 1), date(1999, 7, 15))
s = file_manager.generate_full_file_path(date(1999, 7, 15), date(2005, 1, 1))
split = os.path.split(s)
self.assertEqual(split[0], os.path.dirname(os.path.realpath(__file__)) + "\\downloads")
self.assertEqual(split[1], "research_gov_1999-07-15_2005-01-01.xml")
def test_create_file_versions(self):
file_manager = FileManager()
finder = ItemFinder()
handle = finder.find("/")
user_handle = finder.find("/users/system")
file_version = file_manager.create_file_version(handle.item_id, None, user_handle)
self.assertEquals(file_version, 0)
file_version = file_manager.create_file_version(handle.item_id, None, user_handle)
self.assertEquals(file_version, 1)
file_version = file_manager.create_file_version(handle.item_id, 0, user_handle)
self.assertEquals(file_version, 2)
with self.assertRaises(ServiceException) as cm:
file_version = file_manager.create_file_version(handle.item_id, 4, user_handle)
self.assertEqual(cm.exception.response_code, 409)
self.assertEqual("Unknown previous version: 4", cm.exception.message)
def end_upload_data(request):
log.debug("Receiving an end_upload_data order")
if not request.POST:
raise Http404
form = EndUploadForm(request.POST, request.FILES)
if not form.is_valid():
error_msg = _(u'ERROR: Final de importacion no valida')
log.error (error_msg)
return HttpResponseForbidden()
file_manager = FileManager ()
try:
file_manager.upload_end (form.cleaned_data['date'])
except RuntimeError, e:
error_msg = _(u'ERROR: Finalizacion upload con error %s') %e
log.error (error_msg)
return HttpResponse(e.message, status=400)
def do_cleanup_file_name(input_file_name, config, debug):
try:
thetvdb = TheTvDb(config, debug)
database = Database(config, debug)
file_manager = FileManager(config, database, thetvdb, debug)
match = file_manager.match_file(input_file_name)
if match is None:
print "File name '%s' doesn't match any series on the watch list." % input_file_name,
return 2
else:
(file_name, series, episode) = match
print file_manager.get_library_file_name(file_name, episode)
return 0
except:
traceback.print_exc()
return 11
def do_movie_name(input_file_name, config, debug):
try:
moviedb = MovieDB(config, debug)
thetvdb = TheTvDb(config, debug)
database = Database(config, debug)
file_manager = FileManager(config, database, thetvdb, moviedb, debug)
match = file_manager.match_movie_file(input_file_name, True)
if match is None:
print "File name '%s' doesn't match any movies." % input_file_name,
return 2
else:
(file_name, movie, disc) = match
print file_manager.get_movie_library_file_name(file_name, movie, disc)
return 0
except:
traceback.print_exc()
return 11
def upload_data_file(request):
"""
Recibe un fichero via post.
Sólo se debe tragar los ficheros recibidos si no hay importaciones por realizar.
Para averiguar si hay importaciones por realizar hay que revisar todos los directorios de datos
y verificar que todos tengan un "log.txt" en su interior. Si hay algún directorio que no tiene
el fichero no se admitirán nuevas importaciones.
Hay que ferificar la integridad de los ficheros utilizando el valor del md5 que se envÃa en el post.
1. Nombre de usuario y contraseña inválidos:
1. return 403 Forbidden
2. Si hay directorios pendientes de procesar en la cola de importaciones
1. return status 400 Bad Request: The request contains bad syntax or cannot be fulfilled. FIN
3. Calcular md5 al fichero recibido
4. Si los md5 no son iguales
1. return status 409 Conflict: Indicates that the request could not be processed because
of conflict in the request. FIN
5. Crear si no existe un directorio utilizando la fecha enviada con el patrón '%y-%m-%d_%H:%M'
6. Mover el fichero al directorio
7. return 200
"""
log.debug("Receiving an upload_data_file order.")
if not request.POST:
raise Http404
form = UploadFileForm(request.POST, request.FILES)
if not form.is_valid():
log.error(str(form.errors))
return HttpResponseForbidden()
import_queue = ImportQueue ()
if import_queue.todo:
error_msg = _(u'ERROR: Quedan importaciones pendientes')
log.error (error_msg)
return HttpResponse(error_msg, status=400)
file_manager = FileManager ()
try:
file_manager.save_post_file (form.cleaned_data)
except ValueError, e:
return HttpResponse(e.message, status=409)
def do_process_movies(config, debug, move):
thetvdb = TheTvDb(config, debug)
moviedb = MovieDB(config, debug)
database = Database(config, debug)
file_manager = FileManager(config, database, thetvdb, moviedb, debug)
lock = file_manager.get_process_lock()
if lock is None:
print "Another instance of media_library_manager is currently processing. Exiting."
return 3
try:
movie_input_path = config.getLibraryMovieInputPath()
movie_path = config.getLibraryMoviePath()
movie_genre_path = config.getLibraryMovieGenrePath()
if movie_input_path is None:
print "Configuration setting 'movie_input_path' in section 'Library' is missing. Will not attempt processing of movies."
elif movie_path is None:
print "Configuration setting 'movie_path' in section 'Library' is missing. Will not attempt processing of movies."
else:
process_movies(movie_input_path, movie_path, movie_genre_path, file_manager, debug, move)
file_manager.cleanup_recent_folder()
except:
traceback.print_exc()
return 11
finally:
file_manager.relinquish_process_lock()
def __init__(self):
self.fm = FileManager()
#Default Configurations
self.DEBUG = True
self.HOST = socket.gethostname()
self.PORT = 1234
self.TIMEOUT = 30
self.receiving_file = False
#Load Configurations
self.load_config("config/client.conf")
def do_generate(input_directory, config, debug):
if not os.path.exists(input_directory):
print "Directory '%s' does not exist. Exiting." % input_directory,
return 1
try:
thetvdb = TheTvDb(config, debug)
database = Database(config, debug)
file_manager = FileManager(config, database, thetvdb, debug)
files = []
for root, dir, files_to_add in os.walk(input_directory):
for file in files_to_add:
if file_manager.is_media_file(file):
files.append((root, file))
for (root, file) in files:
generate_for_file(root, file)
return 0
except:
traceback.print_exc()
return 11
def test_most_recent_download_date(self):
test_directory = (
os.path.dirname(os.path.realpath(__file__))
+ "\\test_"
+ datetime.datetime.today().strftime("%Y%m%d%H%M%S%f")
)
file_manager = FileManager(test_directory)
self.assertEqual(os.path.isdir(test_directory), False) # Make sure directory doesn't already exist
self.assertEqual(
file_manager.get_most_recent_download_date(), None
) # Directory doesn't exist. Should return None.
os.mkdir(test_directory)
self.assertEqual(file_manager.get_most_recent_download_date(), None) # Directory is empty. Should return None.
test_filename = test_directory + "\\this_is_a_test.xml"
open(test_filename, "a").close()
self.assertEqual(
file_manager.get_most_recent_download_date(), None
) # Directory with no properly formatted file names. Should return None.
test_filename = test_directory + "\\research_gov_2001-07-02_2005-01-30.xml"
open(test_filename, "a").close()
self.assertEqual(
file_manager.get_most_recent_download_date(), date(2005, 1, 30)
) # Only one valid file name. Make sure the date is what we expected.
test_filename = test_directory + "\\research_gov_2001-07-02_2006-02-28.xml"
open(test_filename, "a").close()
test_filename = test_directory + "\\research_gov_2001-07-02_2002-03-14.xml"
open(test_filename, "a").close()
test_filename = test_directory + "\\research_gov_2001-07-02_2006-02-27.xml"
open(test_filename, "a").close()
self.assertEqual(
file_manager.get_most_recent_download_date(), date(2006, 2, 28)
) # Several valid file names. Make sure the correct date is returned.
# Clean up. Can also use shutil.rmtree, but that's a little scary. NOTE that if any test fails, clean up will
# not occur...
os.remove(test_directory + "\\this_is_a_test.xml")
os.remove(test_directory + "\\research_gov_2001-07-02_2005-01-30.xml")
os.remove(test_directory + "\\research_gov_2001-07-02_2006-02-28.xml")
os.remove(test_directory + "\\research_gov_2001-07-02_2002-03-14.xml")
os.remove(test_directory + "\\research_gov_2001-07-02_2006-02-27.xml")
os.rmdir(test_directory)
def __init__(self, window):
self.window = window
self.viewer = window.viewer
self.editor = window.editor
self.about_window = window.about_window
self.language_doc_window = window.language_doc_window
self.help_window = window.help_window
self.files = FileManager()
self.window_expanded = False
self.core_controls = QWidget(window)
self.core_layout = QGridLayout()
self.core_layout.setSpacing(10)
self.more_controls = QWidget(window)
self.more_layout = QVBoxLayout()
self.more_layout.setAlignment(Qt.AlignTop)
self.more_layout.setSpacing(10)
self.setup_controls()
def do_process(config, debug, move):
thetvdb = TheTvDb(config, debug)
database = Database(config, debug)
file_manager = FileManager(config, database, thetvdb, debug)
lock = file_manager.get_process_lock()
if lock is None:
print "Another instance of media_library_manager is currently processing. Exiting."
return 3
else:
try:
input_path = config.getLibraryInputPath()
tv_path = config.getLibraryTvPath()
if not os.path.exists(input_path):
print "Input path '%s' does not exist. Exiting." % input_path,
return 1
if not os.path.exists(tv_path):
print "Library TV path '%s' does not exist. Exiting." % tv_path,
return 2
files = []
for root, dir, files_to_add in os.walk(input_path):
for file in files_to_add:
if file_manager.is_media_file(file):
files.append((root, file))
input_contents = os.listdir(input_path)
for (root, file) in files:
file_path = os.path.join(root, file)
if os.path.isfile(file_path):
process_file(file_manager, file_path, tv_path, debug, move)
file_manager.cleanup_recent_folder()
except:
traceback.print_exc()
return 11
finally:
file_manager.relinquish_process_lock()
def do_process_tv(config, debug, move):
thetvdb = TheTvDb(config, debug)
moviedb = MovieDB(config, debug)
database = Database(config, debug)
file_manager = FileManager(config, database, thetvdb, moviedb, debug)
lock = file_manager.get_process_lock()
if lock is None:
print "Another instance of media_library_manager is currently processing. Exiting."
return 3
try:
input_path = config.getLibraryInputPath()
tv_path = config.getLibraryTvPath()
tv_genre_path = config.getLibraryTvGenrePath()
process_tv(input_path, tv_path, tv_genre_path, file_manager, debug, move)
file_manager.cleanup_recent_folder()
except:
traceback.print_exc()
return 11
finally:
file_manager.relinquish_process_lock()
#! /usr/bin/env python # -*- coding: utf-8 -*- from deduper import Deduper from config import file_name, fields from file_manager import FileManager from generator import Generator # Generate input data if it does not exist input_generator = Generator() input_generator.generate_data(file_name['input']) # Create deduper deduper = Deduper(file_name['setting'], fields) # Get data dictionary data_file = FileManager(file_name['input']) data_sample = data_file.data_dict # Train deduper.train(data_sample, True, file_name['training']) # Active Labeling deduper.start_active_labeling(file_name['training'], file_name['setting']) # Clustering deduper.start_clustering(data_sample) # Writing output output_file = FileManager(file_name['output'],'w') output_file.out(file_name['input'], deduper.clusters)
class Controls():
default_animation_speed_slider_value = 50
default_brush_size_slider_value = 1
def __init__(self, window):
self.window = window
self.viewer = window.viewer
self.editor = window.editor
self.about_window = window.about_window
self.language_doc_window = window.language_doc_window
self.help_window = window.help_window
self.files = FileManager()
self.window_expanded = False
self.core_controls = QWidget(window)
self.core_layout = QGridLayout()
self.core_layout.setSpacing(10)
self.more_controls = QWidget(window)
self.more_layout = QVBoxLayout()
self.more_layout.setAlignment(Qt.AlignTop)
self.more_layout.setSpacing(10)
self.setup_controls()
def setup_controls(self):
# 'Run' btn: runs user's code from editor
self.run_btn = QPushButton('Run', self.window)
self.run_btn.clicked.connect(self.run_btn_clicked)
self.core_layout.addWidget(self.run_btn, 0, 0)
# 'New' btn: clear editor and viewer areas
self.new_btn = QPushButton('New', self.window)
self.new_btn.clicked.connect(self.new_btn_clicked)
self.core_layout.addWidget(self.new_btn, 1, 0)
# 'Save' btn: saves editor contents
self.save_btn = QPushButton('Save', self.window)
self.save_btn.clicked.connect(self.save_btn_clicked)
self.core_layout.addWidget(self.save_btn, 0, 1)
# 'Load' btn: loads contents from .txt file to editor
self.load_btn = QPushButton('Load', self.window)
self.load_btn.clicked.connect(self.load_btn_clicked)
self.core_layout.addWidget(self.load_btn, 1, 1)
# 'More' btn: opens panel containing more controls
self.more_btn = QPushButton('More\n>>>', self.window)
self.more_btn.clicked.connect(self.more_btn_clicked)
self.core_layout.addWidget(self.more_btn, 0, 2, 2, 1)
# Coordinate labels
self.mouse_pos = QLabel(self.window)
self.mouse_pos.setSizePolicy(QSizePolicy.Preferred,
QSizePolicy.Expanding)
self.more_layout.addWidget(self.mouse_pos)
# 'BG Color' btn: changes viewer background color
self.bgcolor_btn = QPushButton('BG Color', self.window)
self.bgcolor_btn.setSizePolicy(QSizePolicy.Preferred,
QSizePolicy.Expanding)
self.bgcolor_btn.clicked.connect(self.bgcolor_btn_clicked)
self.more_layout.addWidget(self.bgcolor_btn)
# 'FG Color' btn: changes viewer foreground color
self.fgcolor_btn = QPushButton('FG Color', self.window)
self.fgcolor_btn.setSizePolicy(QSizePolicy.Preferred,
QSizePolicy.Expanding)
self.fgcolor_btn.clicked.connect(self.fgcolor_btn_clicked)
self.more_layout.addWidget(self.fgcolor_btn)
# Animation Speed slider: changes animation speed
self.anim_speed_controller = AnimSpeedController(self.window)
self.more_layout.addWidget(self.anim_speed_controller)
# Brush Size slider: changes brush size
self.brush_size_controller = BrushSizeController(self.window)
self.more_layout.addWidget(self.brush_size_controller)
# 'Language Documentation' btn: opens a window containing information
# directing users to where they can find documentation about commands
self.lang_doc_btn = QPushButton('Language Documentation', self.window)
self.lang_doc_btn.setSizePolicy(QSizePolicy.Preferred,
QSizePolicy.Expanding)
self.more_layout.addWidget(self.lang_doc_btn)
self.lang_doc_btn.clicked.connect(self.lang_doc_btn_clicked)
# 'Load Tutorial Code' btn: loads code from tutorial into editor
self.tutorial_btn = QPushButton('Load Tutorial Code', self.window)
self.tutorial_btn.setSizePolicy(QSizePolicy.Preferred,
QSizePolicy.Expanding)
self.more_layout.addWidget(self.tutorial_btn)
self.tutorial_btn.clicked.connect(self.load_tutorial_code_btn_clicked)
# 'Help' btn: opens window containing information directing users to where
# they can find the 'online help' documentation
self.help_btn = QPushButton('Help', self.window)
self.help_btn.setSizePolicy(QSizePolicy.Preferred,
QSizePolicy.Expanding)
self.more_layout.addWidget(self.help_btn)
self.help_btn.clicked.connect(self.help_btn_clicked)
# 'About' btn: opens window containing information about this application
self.about_btn = QPushButton("About", self.window)
self.about_btn.setSizePolicy(QSizePolicy.Preferred,
QSizePolicy.Expanding)
self.about_btn.clicked.connect(self.about_btn_clicked)
self.more_layout.addWidget(self.about_btn)
# Core Controls
self.core_controls.setLayout(self.core_layout)
self.core_controls.setGeometry(470, 405, 330, 75)
# More Controls
self.more_controls.setLayout(self.more_layout)
self.more_controls.setGeometry(790, 0, 210, 480)
# Control Functionality
def new_btn_clicked(self):
if self.editor.is_modified():
confirmation = ConfirmNew()
state = confirmation.get_state()
if state is 'yes':
# Yes button was clicked
self.files.clear_fname()
self.editor.clear()
self.viewer.clear()
else:
# No button or nothing was clicked
pass # Do nothing
else:
self.files.clear_fname()
self.editor.clear()
self.viewer.clear()
def run_btn_clicked(self):
commands = self.editor.tokenise(self.viewer)
self.viewer.render(commands)
def save_btn_clicked(self):
self.files.save(self.window, self.editor)
def load_btn_clicked(self):
self.files.load(self.window, self.editor)
def more_btn_clicked(self):
sender, new_width = self.window.sender(), None
if not self.window_expanded:
self.window_expanded = True
sender.setText("Less\n<<<")
new_width = 1000
else:
self.window_expanded = False
sender.setText("More\n>>>")
new_width = 800
self.window.setFixedSize(new_width, 480)
def bgcolor_btn_clicked(self):
bg_color = QColorDialog.getColor()
if bg_color.isValid():
self.viewer.set_bg_color(bg_color)
def fgcolor_btn_clicked(self):
fg_color = QColorDialog.getColor()
if fg_color.isValid():
self.viewer.set_fg_color(fg_color)
def about_btn_clicked(self):
self.about_window.show()
def lang_doc_btn_clicked(self):
self.language_doc_window.show()
def load_tutorial_code_btn_clicked(self):
self.files.load_tutorial_file(self.window, self.editor)
def help_btn_clicked(self):
self.help_window.show()
def update_mouse_positional_label(self, window):
window_pos = window.pos()
mouse_pos = QCursor().pos()
mx = mouse_pos.x() - window_pos.x() - 10
my = mouse_pos.y() - window_pos.y() - 33
if mx > 460 or my > 460 or mx < 0 or my < 0:
mx, my = "-", "-"
self.mouse_pos.setText("Mouse: {} , {}".format(mx, my))
class BWClient:
def __init__(self):
self.fm = FileManager()
#Default Configurations
self.DEBUG = True
self.HOST = socket.gethostname()
self.PORT = 1234
self.TIMEOUT = 30
self.receiving_file = False
#Load Configurations
self.load_config("config/client.conf")
def load_config(self,url):
config = self.fm.load_json(url)
if None == config:
self.write_config(url)
if self.DEBUG:
print("Configuration file created. Defaults loaded.")
return False
if config["DEBUG"]:
self.DEBUG = config["DEBUG"]
if config["HOST_NAME"]:
self.HOST = config["HOST_NAME"]
if config["PORT"]:
self.PORT = config["PORT"]
if config["SOCKET_TIMEOUT"]:
self.TIMEOUT = config["SOCKET_TIMEOUT"]
if self.DEBUG:
print("Configurations loaded.")
return True
def write_config(self,url):
data = {
"DEBUG":self.DEBUG,
"HOST_NAME":None,
"PORT":None,
"SOCKET_TIMEOUT":self.TIMEOUT}
self.fm.write_json(url,data)
def set_address(self,host,port):
self.HOST = host
self.PORT = port
def connect(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.DEBUG:
print ("Socket created.")
connected = False
while not connected:
try:
self.sock.connect((self.HOST, self.PORT))
connected = True
if self.DEBUG:
print ("Socket connected.")
except socket.error as msg:
if self.DEBUG:
print ("Connection failed.")
sleep(5)
self.sock.setblocking(0)
def reconnect(self):
self.sock.close()
if self.DEBUG:
print ("Socket closed.")
self.connect()
def listen(self):
void_messages = 0;
if not self.receiving_file:
try:
data = self.sock.recv(1024)
if data:
str_data = str(data,'UTF-8')
if self.DEBUG:
print (str_data)
if str_data == "Pong":
void_messages = 0
sleep(1)
if str_data == "File":
self.receiving_file = True;
self.fm.receive_file(self.sock,"wpt.bmp")
except socket.error as msg:
if(void_messages > self.TIMEOUT):
if self.DEBUG:
print ("Connection timed out.")
self.reconnect()
else:
try:
self.sock.send(bytes("Ping",'UTF-8'))
if self.DEBUG:
print("Ping")
except socket.error as msg:
if self.DEBUG:
print ("Connection to server lost.")
self.reconnect()
sleep(1)
def start(self):
self.connect()
while True:
self.listen()
def __init__(self): self.fm = FileManager() self.DEBUG = True self.HOST = socket.gethostname() self.PORT = 1234 self.LISTEN_QUEUE = 10
class BWServer:
def __init__(self):
self.fm = FileManager()
self.DEBUG = True
self.HOST = socket.gethostname()
self.PORT = 1234
self.LISTEN_QUEUE = 10
def start(self):
self.bind()
while True:
self.listen()
def bind(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.DEBUG:
print ("Socket created.")
try:
self.sock.bind((self.HOST, self.PORT))
print ("Socket bound successfully")
except socket.error as msg:
if self.DEBUG:
print ("Bind failed.",msg)
sys.exit()
self.sock.listen(self.LISTEN_QUEUE)
def listen(self):
client, client_addr = self.sock.accept()
if self.DEBUG:
print (str(client_addr[0]) + ":" + str(client_addr[1]), "- Client connected")
start_new_thread(self.communicate,(client,client_addr))
#I am a threaded function
def communicate(self, client, client_addr):
client.send(bytes("Welcome to the server!",'UTF-8'))
sleep(1)
while True:
try:
data = client.recv(1024)
if not data:
break
command = str(data,'UTF-8')
print(command)
reply = ""
if "Ping" == command or "PingPing" == command:
reply = "File"
print(reply)
client.send(bytes(reply,'UTF-8'))
self.fm.send_file(client,"Wallpaper_Teacher.bmp")
except socket.error as msg:
#We lost them
break;
client.close()
if self.DEBUG:
print (str(client_addr[0]) + ":" + str(client_addr[1]), "- Client disconnected")