def deploy():
"""
Deploys remotely master branch changes to production server.
Returns:
None
"""
try:
print()
info_print(
"This script will only succeed if you have configured the "
"production server ssh port to match the one in "
"config/production.py and have proper ssh access."
)
warning_print(
"This process will update the production server with the "
"latest master branch changes."
)
option = input("Do you want to continue? (y/n)\n")
if option != "y" and option != "Y":
sys.exit()
t0 = time.time()
remote_production_task("update_project")
remote_production_task("install_python_requirements")
remote_production_task("collect_static_files")
remote_production_task("make_migrations")
remote_production_task("migrate")
remote_production_task("restart_server")
t1 = time.time()
show_time(t0, t1)
except KeyboardInterrupt:
error_print("Process aborted. Bye!")
sys.exit()
def test_unpack(self):
"""
creates test folder, archives it and tests for custom unpack to work as expected
"""
target_archive = "./test/test.archive"
create_test_folders("test_folder")
target_files = ["/file1", "/inside_dir/file4"]
unpacked_files = []
no_error = True
ok = True
try:
run_create_archive(target_archive, "./test/test_folder")
run_unpack(target_archive, "./test/archived_folder", *target_files)
unpacked_files = [
x.replace("\\", "/")
for x in get_local_path_of_files_from_directory(
"./test/archived_folder")
]
except Exception as e:
error_print(e)
no_error = False
shutil.rmtree("./test")
self.assertTrue(no_error, "Runtime error")
self.assertEqual(target_files, unpacked_files,
"Unpacked files are not the expected ones")
def test_create_archive_from_files(self):
"""
creates test folder, runs create_folder on some files, full unpack and verifies equality of used files
should runtime error occur, test fails
"""
target_archive = "./test/test.archive"
create_test_folders("test_folder")
# create_test_folders("archived_folder")
target_files = [
"./test/test_folder/file1", "./test/test_folder/inside_dir/file4"
]
no_error = True
ok = True
try:
run_create_archive(target_archive, *target_files)
run_full_unpack(target_archive, "./test/archived_folder")
unpacked_files = get_files_from_directory("./test/archived_folder")
if len(target_files) != len(unpacked_files):
ok = False
for i, j in zip(target_files, unpacked_files):
if not are_files_equal(i, j):
ok = False
except Exception as e:
error_print(e)
no_error = False
shutil.rmtree("./test")
self.assertTrue(no_error, "Runtime error")
self.assertTrue(ok, "Content of files are not equal")
def remote_development_task(task_name):
"""
Executes remotely a single task on the development machine.
Returns:
None
"""
try:
function = getattr(development, task_name, "doesnt_exist_in_dev")
if function == "doesnt_exist_in_dev":
function = getattr(base, task_name)
if not inspect.isfunction(function):
raise AttributeError
cmd = "vagrant ssh -c \"" + function(mode="dev") + "\""
os.chdir("vagrant")
run(cmd)
os.chdir("../")
except AttributeError:
error_print(
"The command '" + task_name +
"' you are trying to execute doesn't exist."
)
except KeyboardInterrupt:
error_print("Process aborted. Bye!")
sys.exit()
def remote_production_task(task_name):
"""
Executes remotely a single task on the production server.
Returns:
None
"""
try:
function = getattr(production, task_name, "doesnt_exist_in_prod")
if function == "doesnt_exist_in_prod":
function = getattr(base, task_name)
if not inspect.isfunction(function):
raise AttributeError
cmd = (
"ssh root@" + ip + " -p " + port + " \"" +
function(mode="prod") + "\""
)
run(cmd)
except AttributeError:
error_print(
"The command '" + task_name +
"' you are trying to execute doesn't exist."
)
except KeyboardInterrupt:
error_print("Process aborted. Bye!")
sys.exit()
def _print_read_page_error(self, title, text, reason=""):
if WikiCreator.DEBUG:
after = self._wiki_file.read(100)
utils.error_print("Error when reading: " + title + "." + "at: " + str(self._titles[title][0]) + " reason: "\
+ reason + "\n100 chars before:\n")
self._wiki_file.seek(self._titles[title][0] - 100)
utils.error_print(self._wiki_file.read(100) + "\npage: ")
utils.error_print(text)
utils.error_print("\n100 after:\n")
utils.error_print(after + "\ndone\n")
def setup_development_machine():
"""
Creates remotely a new ubuntu virtual machine ready to work on the project.
Returns:
None
"""
try:
print()
info_print("You need to have Vagrant installed in your host machine.")
warning_print(
"This process will clean all data on your previous virtual "
"machine and build it again from scratch."
)
option = input("Do you want to continue? (y/n)\n")
if option != "y" and option != "Y":
sys.exit()
t0 = time.time()
pretty_print("\nCleaning previous installation ...")
os.chdir("vagrant")
run("vagrant destroy --force")
run("vagrant box update")
run("vagrant up")
os.chdir("../")
remote_development_task("update_repositories")
remote_development_task("create_app_group_and_user")
remote_development_task("install_postgresql")
remote_development_task("create_postgresql_user_and_db")
remote_development_task("install_jpeg_libraries_for_pillow")
remote_development_task("install_git")
remote_development_task("download_project")
remote_development_task("install_virtualenv")
remote_development_task("create_virtualenv")
remote_development_task("export_project_keys")
remote_development_task("install_python_requirements")
remote_development_task("make_migrations")
remote_development_task("migrate")
remote_development_task("install_npm_and_grunt")
remote_development_task("install_npm_packages")
remote_development_task("install_ruby_and_sass")
remote_development_task("install_unity")
remote_development_task("create_test_data")
remote_development_task("install_pycharm")
remote_development_task("configure_pycharm")
os.chdir("vagrant")
run("vagrant halt")
os.chdir("../")
local_development_task("show_last_tips")
t1 = time.time()
show_time(t0, t1)
except KeyboardInterrupt:
error_print("Process aborted. Bye!")
sys.exit()
def ssh_production_server():
"""
Connects remotely to the production server through ssh.
Returns:
None
"""
try:
run("ssh root@" + ip + " -p " + port)
except KeyboardInterrupt:
error_print("Process aborted. Bye!")
sys.exit()
def setup_production_server():
"""
Configures remotely the production server initial setup and deployment.
Returns:
None
"""
try:
print()
info_print(
"This script will only succeed if you have configured the "
"production server ssh port to match the one in "
"config/production.py and have proper ssh access."
)
info_print(
"After this setup, you should configure the admin allowed IP "
"in the nginx configuration file."
)
warning_print(
"This process will clean all data on the production server "
"and build up everything again from scratch."
)
option = input("Do you want to continue? (y/n)\n")
if option != "y" and option != "Y":
sys.exit()
t0 = time.time()
remote_production_task("clean_production_server")
remote_production_task("update_repositories")
remote_production_task("create_app_group_and_user")
remote_production_task("install_postgresql")
remote_production_task("create_postgresql_user_and_db")
remote_production_task("install_jpeg_libraries_for_pillow")
remote_production_task("install_git")
remote_production_task("download_project")
remote_production_task("install_virtualenv")
remote_production_task("create_virtualenv")
remote_production_task("export_project_keys")
remote_production_task("install_python_requirements")
remote_production_task("create_runtime_files_and_dirs")
remote_production_task("collect_static_files")
remote_production_task("make_migrations")
remote_production_task("migrate")
remote_production_task("install_supervisor")
remote_production_task("configure_supervisor")
remote_production_task("install_nginx")
remote_production_task("configure_nginx")
remote_production_task("restart_server")
t1 = time.time()
show_time(t0, t1)
except KeyboardInterrupt:
error_print("Process aborted. Bye!")
sys.exit()
def test():
x, y = utils.read_file(is_train=True, label_list=['人类作者', '自动摘要'])
x = utils.process(x)
x = utils.truncation(x)
word2id, id2word, tag2id, id2tag = utils.build_vocab(x, y, min_df=10)
x = utils.build_x_ids(x, word2id)
y = utils.build_y_ids(y, tag2id)
data = zip(x, y)
train_data, dev_data = train_test_split(data,
test_size=10000,
random_state=24)
vocab_size = len(word2id)
emb_dim = 100
num_classes = len(tag2id)
print "训练集数据大小:%d 验证集数据大小:%d" % (len(train_data), len(dev_data))
print "vocab_size:%d num_classes:%d" % (vocab_size, num_classes)
print FLAGS.model_name
model_dir = os.path.join('temp', 'nn')
if not os.path.exists(model_dir):
os.mkdir(model_dir)
with tf.Session() as sess:
model = getattr(models, FLAGS.model_name)(vocab_size, emb_dim,
num_classes)
saver = tf.train.Saver(tf.global_variables())
model_file = os.path.join('temp', 'nn', FLAGS.model_file)
saver.restore(sess, model_file)
print "Restore model from %s" % model_file
dev_loss = []
labels = []
predicts = []
bar = ProgressBar(max_value=len(dev_data) // FLAGS.batch_size + 1)
for batch_data in bar(
utils.minibatches(dev_data,
FLAGS.batch_size,
True,
shuffle=False)):
loss, predict = model.dev_step(sess, batch_data)
dev_loss.append(loss)
labels.extend(batch_data[1])
predicts.extend(predict)
dev_loss = np.mean(dev_loss)
dev_f1 = utils.score_all(labels, predicts, tag2id)
utils.error_print(predicts, labels, id2tag, zip(*dev_data)[0], id2word)
print "loss:%.3f f1:%.3f" % (dev_loss, dev_f1)
def main() -> None:
load_dotenv()
token = os.getenv('TOKEN')
raw_link = get_args().url
if not raw_link:
error_print('Your link is empty. Try again.')
exit()
try:
protocol, link = raw_link.split('://') \
if 'http' in raw_link else ('http', raw_link)
except ValueError:
error_print('Wrong url format. Try again.')
exit()
try:
if is_bitlink(link, token):
success_print('This link has been clicked {} times.'.format(
get_click_count(link, token)))
else:
success_print('This is your very short link: {}'.format(
get_bitlink(f'{protocol}://{link}', token)))
except requests.exceptions.HTTPError as e:
if e.response.status_code == 400:
error_print(e.response.json()['description'])
else:
error_print(str(e))
def close_bid(self, cat_name, prod_name):
""" Close a bid
:param cat_name: category name
:param prod_name: product name
"""
print "Closing bid " + prod_name
try:
cat = Category.search_category(self._auct._categories, cat_name)
auct = cat.search_auction(prod_name)
auct.close(self.user)
cat.del_auction(auct)
self.response(1)
except CategoryException, e:
# Not matching category
error_print(e)
self.response(0)
def _get_page(self, title):
if title not in self._titles:
return None
# TODO: this is a fix for page not ending correctly as pythons read reads chars and not bytes (i.e. we were of
# because of the encoding). this fix isnt good performance wise so if you have time change it.
self._wiki_file.seek(self._titles[title][0])
res = self._wiki_file.read(self._titles[title][1]).strip()
start_line = res.splitlines()[0]
end_line = res.splitlines()[-1]
if (not start_line.strip().startswith(u'<page>')) or \
((not end_line.strip().endswith(u'</page>\n')) and (not end_line.strip().endswith(u'</page>'))):
utils.error_print("u'</page>\\n' is " + str(not end_line.strip().endswith(u'</page>\n')))
utils.error_print("u'<page>' is " + str((not start_line.strip().startswith(u'<page>'))))
self._print_read_page_error(title, res, "bad page start or end\n start line: " +
start_line + "\n end: " + end_line)
return res
def data_gathering(parameters):
start_date = datetime.datetime.strptime(parameters['days_test_starts'], '%Y-%m-%d')
end_date = datetime.datetime.strptime(parameters['days_test_ends'], '%Y-%m-%d')
# check; where is the data?
data_access_fail, ab_test_total, = 0, pd.DataFrame()
try:
if parameters['is_from_csv']:
data_access_fail += 1
control, validation = pd.read_csv('control.csv'), pd.read_csv('validation.csv')
data_access_fail += 0.1
control['is_control'], validation['is_control'] = 0, 1
ab_test = pd.concat([control, validation])
if not parameters['is_segmented']:
data_access_fail += 0.1
metrics = parameters['metrics'] if parameters['metrics'] != [] else constants.METRICS
ab_test_total = ab_test[constants.AB_TEST_DATAFRAME_KEY_COLUMS + metrics]
else:
data_access_fail += 0.1
metrics = parameters['metrics'] if parameters['metrics'] != [] else constants.METRICS
ab_test_total = ab_test[constants.AB_TEST_DATAFRAME_KEY_COLUMS + metrics +
constants.SEGMENT_DATA_COLUMN]
else:
data_access_fail = 2
if parameters['is_from_db']:
ab_test = get_data_from_db(start_date, end_date)
if not parameters['is_segmented']:
data_access_fail += 0.1
metrics = parameters['metrics'] if parameters['metrics'] != [] else constants.METRICS
ab_test_total = ab_test[constants.AB_TEST_DATAFRAME_KEY_COLUMS + metrics]
else:
data_access_fail += 0.1
ab_test = get_segments(ab_test, constants.LAST_RFM_DATE)
metrics = parameters['metrics'] if parameters['metrics'] != [] else constants.METRICS
ab_test_total = ab_test[constants.AB_TEST_DATAFRAME_KEY_COLUMS + metrics +
constants.SEGMENT_DATA_COLUMN]
else:
data_access_fail = 3
if parameters['is_data_randomly_generated']:
segments = pd.read_csv('segments.csv')
ab_test_total = get_random_ab_test_generator(start_date, end_date, segments, parameters)
data_access_fail += 1
except:
utils.error_print(data_access_fail, parameters)
return ab_test_total
def login(self, user, pwd):
"""
Log a user on the server
:param user: username
:param pwd: MD5 hash of the password
"""
username = user
try:
self.user = self._auct._users[username]
token = ""
if self.user.password == pwd:
# Login effettuato
self.logged = True
self.response(1, token=self.gen_session())
return
debug_print("Password non valida")
self.response(0, "Password non valida", token="")
except KeyError, e:
error_print("Utente non trovato")
self.response(0, "Utente non trovato", token="")
def test_create_archive_from_directory(self):
"""
creates test folder, runs create_archive on it and full_unpack, then verifies equality of initial and unpacked
should runtime error occur, test fails
"""
target_archive = "./test/test.archive"
create_test_folders("test_folder")
# create_test_folders("archived_folder")
no_error = True
ok = True
try:
run_create_archive(target_archive, "./test/test_folder")
# run_create_archive(target_archive, "./utils.rar")
run_full_unpack(target_archive, "./test/archived_folder")
ok = are_folders_equal("./test/archived_folder",
"./test/test_folder")
except Exception as e:
no_error = False
error_print(e)
shutil.rmtree("./test")
self.assertTrue(no_error, "Runtime error")
self.assertTrue(ok, "Folders are not equal")
def main():
args_parser = argparse.ArgumentParser(
description='Use java tool to translate wiki format to text')
args_parser.add_argument('command',
choices=['build', 'run', 'both'],
help='command to run (build or run or both')
args_parser.add_argument('--wiki_file',
'-f',
action='store',
help='path to file of wiki dump format')
args_parser.add_argument('--base',
'-b',
action='store',
default='.',
help='Base folder to work in')
args_parser.add_argument('--outfile',
'-o',
action='store',
default=None,
help='file to write output too')
args_parser.add_argument('--errorfile',
'-e',
action='store',
default=None,
help='file to write errors too')
args = args_parser.parse_args(sys.argv[1:])
# if len(sys.argv) < 2:
# print('usage: python wiki2text_wrapper.py {build | run | build run} <params>')
# print('build will compile the java class into project_folder/bin. optional param: LMBuilder_base="."')
# print('run will not build before running the class on the params, params = "path_to_a_wiki_xml [LMbuilder_base]"')
# print('build run will execute both one after the other with params for run')
# exit()
project_folder = args.base
command = args.command
output = None
if args.outfile is not None:
output = open(args.outfile, 'w', encoding='utf-8')
error_out = None
if args.errorfile is not None:
error_out = open(args.errorfile, 'w', encoding='utf-8')
if command == 'build' or command == 'both':
code = build_wiki2text(project_folder=project_folder,
out=output,
error=error_out)
if code != 0:
utils.error_print('Wiki2Text: build failed\n')
exit(code)
if command == 'run' or command == 'both':
if args.wiki_file is None:
utils.error_print('Wiki2Text: cant run without a wiki file\n')
exit(1)
path_to_wiki = args.wiki_file
code = run_wiki2text(path_to_wiki,
project_folder=project_folder,
out=args.outfile,
error=args.errorfile)
if code != 0:
utils.error_print('Wiki2Text: run error (in java tool)\n')
exit(code)
def local_development_task(task_name):
"""
Executes locally a single task on the development machine.
Returns:
None
"""
try:
function = getattr(development, task_name, "doesnt_exist_in_dev")
if function == "doesnt_exist_in_dev":
function = getattr(base, task_name)
if not inspect.isfunction(function):
raise AttributeError
run(function(mode="dev"))
except AttributeError:
error_print(
"The command '" + task_name +
"' you are trying to execute doesn't exist."
)
except KeyboardInterrupt:
error_print("Process aborted. Bye!")
sys.exit()
def test_list_content(self):
"""
creates test folder, then archives it and verifies list_archive_content to be same as model
"""
target_archive = "./test/test.archive"
create_test_folders("test_folder")
correct_answer = ["/file1", "/file2", "/file3", "/inside_dir/file4"]
list_archive_content = []
no_error = True
ok = True
try:
run_create_archive(target_archive, "./test/test_folder")
list_archive_content = [
x.replace("\\", "/")
for x, y in get_archive_content(target_archive)[1]
]
except Exception as e:
error_print(e)
no_error = False
shutil.rmtree("./test")
self.assertTrue(no_error, "Runtime error")
self.assertEqual(correct_answer, list_archive_content,
"Listed content is not the same")
exit(0)
try:
target_files = sys.argv[4:]
except IndexError:
raise CustomError("Target files not mentioned")
if command == "-unpack":
run_unpack(target_archive, target_folder, *target_files)
exit(0)
raise CustomError("Invalid first argument. Run `python a_seven.py --help` for help")
if __name__ == "__main__":
# sys.argv[1:] = ["--help"]
# sys.argv[1:] = ["-create_archive", "./test.archive", "./utils"]
# sys.argv[1:] = ["-list_content", "./test.archive"]
# sys.argv[1:] = ["-create_archive", "./test2.archive", "./LICENSE", "./test.archive"]
# sys.argv[1:] = ["-full_unpack", "./test2.archive", "./test_unpack_2"]
# sys.argv[1:] = ["-unpack", "./test.archive", "./test_unpack_4", "/custom_error.py", "/open_2.py"]
if len(sys.argv) < 2:
error_print("Invalid syntax. Run `python a_seven.py --help` for help")
try:
run(sys.argv[1])
except CustomError as e:
error_print(f"Error: {e}")
exit("Error")
except Exception as e:
error_print(f"Other error: {e}")
exit("Error 2")
def library(library, skipnoalbum=False):
for track in library:
title = track["Name"]
tokens = []
location = urllib.request.unquote(track["Location"][7:].replace(
"%20", " "))
filename, file_extension = os.path.splitext(location)
utils.notice_print(title)
if os.path.exists(location):
if file_extension == ".mp3":
audio = MP3(location, ID3=EasyID3)
audio["artistsort"] = ['']
audio["titlesort"] = ['']
audio["albumsort"] = ['']
if "title" in audio.keys():
if audio["title"][0] != title:
utils.warning_print('different titles\n\tTitle: ' +
str(audio["title"][0]) + "\n" +
'\tName: ' + title)
title_selection = utils.prompt(
'Which one would you like to keep? ',
["1", "2", "s"], 0)
if title_selection == "2":
audio["title"] = [title]
elif title_selection == "1":
title = audio["title"][0]
else:
utils.warning_print("no title")
title_duplication = utils.prompt(
'Would you like to clone the name? ', ["1", "2"], 0)
if title_duplication == "1":
audio["title"] = [title]
elif title_duplication == "2":
audio["title"] = []
if "artist" in audio.keys():
if "albumartist" in audio.keys():
if audio["artist"] != audio["albumartist"]:
utils.warning_print(
'different artists\n\tArtist: ' +
str(audio["artist"]) + "\n" +
'\tAlbum artist: ' + str(audio["albumartist"]))
artist_selection = utils.prompt(
'Which one would you like to keep? ',
["1", "2", "s"], 2)
if artist_selection == "1":
audio["albumartist"] = audio["artist"]
elif artist_selection == "2":
audio["artist"] = audio["albumartist"]
else:
utils.warning_print("no album artist")
if skipnoalbum is False:
artist_duplication = utils.prompt(
'Would you like to substitute < no album artist > with < '
+ audio["artist"][0] + ' >? ', ["1", "2", "s"],
1)
if artist_duplication == "1":
audio["albumartist"] = audio["artist"]
elif artist_duplication == "2":
audio["albumartist"] = []
else:
utils.warning_print("no artist")
title_components = title.split("-")
title_components_dir = {}
i = 0
noconflicts = {"title": False, "artist": False}
if len(title_components) > 1:
utils.warning_print("splittable name")
for comp in title_components:
comp = comp.strip()
title_components_dir[i] = comp
if "title" in audio.keys() and len(
audio["title"]) > 0 and comp.lower(
) == audio["title"][0].lower():
noconflicts["title"] = comp
del title_components_dir[i]
if "artist" in audio.keys() and len(
audio["artist"]) > 0 and comp.lower(
) == audio["artist"][0].lower():
noconflicts["artist"] = comp
del title_components_dir[i]
print("\t" + str(i + 1) + " - " + comp)
i += 1
# print (len(title_components_dir))
title_components_keys = list(title_components_dir.keys())
if len(title_components_keys) == 1:
suggestion = 0
else:
suggestion = False
if noconflicts["title"] is False:
newtitle = utils.prompt(
'Which term is the title? ',
[str(val + 1)
for val in title_components_keys] + ["s"],
suggestion)
if newtitle != "s":
audio["title"] = title_components[int(newtitle) -
1].strip()
title_components_keys.remove(int(newtitle) - 1)
if len(title_components_keys) == 1:
suggestion = 0
else:
suggestion = False
if noconflicts["artist"] is False:
newartist = utils.prompt(
'Which term is the artist? ',
[str(val + 1)
for val in title_components_keys] + ["s"],
suggestion)
if newartist != "s":
audio["artist"] = title_components[int(newartist) -
1].strip()
audio.save()
else:
utils.error_print("Wrong file extension. Extension: " +
file_extension)
else:
utils.error_print("File not found. Location: " + location)
utils.notice_print("All done.\n\n")
else:
# try:
if len(sys.argv) < 3:
raise CustomError(
"Script does not have enough arguments. Run `python main.py --help` for help"
)
input_file = sys.argv[1]
similarity_function = sys.argv[2]
if similarity_function not in similarity_functions:
raise CustomError(
f"Second argument should be one of the following: {', '.join(similarity_functions)}"
)
all_solutions = False
try:
if sys.argv[3] == "--all-solutions":
all_solutions = True
except IndexError:
pass
prepare_clustering(input_file, similarity_function, all_solutions)
# except CustomError as e:
# error_print(f"Error: {e}")
# except Exception as e:
# error_print(f"Other error: {e}")
if __name__ == "__main__":
if len(sys.argv) < 2:
error_print(
"Run the program as following for help:\npython main.py --help")
run()
:param prod_name: product name
"""
print "Closing bid " + prod_name
try:
cat = Category.search_category(self._auct._categories, cat_name)
auct = cat.search_auction(prod_name)
auct.close(self.user)
cat.del_auction(auct)
self.response(1)
except CategoryException, e:
# Not matching category
error_print(e)
self.response(0)
except AuctionException, e:
# No Winner
error_print(e)
self.response(8)
except ExistingAuctionException, e:
#No auction
error_print(e)
self.response(5)
except UserException, e:
#User not allowed to close
error_print(e)
self.response(7)
def response(self, response, res_msg="", token={}):
"""Compose a generic response
:param response: response code
:param res_msg: response message
:param token: optional, token of the session