def deploy(deploy, cfg_file):
cfg = load_config.get_config(cfg_file)
date_string = datetime.datetime.today().strftime("%Y-%m-%d_%H-%M-%S")
timestamp = int(time.time())
file_count = sum([len(files) for r, d, files in os.walk(deploy)])
latest_deploy = str(date_string) + "\n" + str(timestamp) + "\n" + str(
file_count)
cmd = 'aws s3 sync ' + deploy + '/ s3://' + cfg['BUCKET'] + '/' + cfg[
'NICKNAME'] + '/' + date_string + '/ --only-show-errors'
log("Running the following shell command to sync files: '" + cmd + "'")
subprocess.run(cmd, check=True, shell=True)
log("Writing the following file to S3 to trigger deploy on instances:\n" +
latest_deploy)
client = boto3.client('s3')
client.put_object(Bucket=cfg['BUCKET'],
Key=cfg['NICKNAME'] + '/latest-deploy',
Body=latest_deploy)
s3 = boto3.resource('s3')
s3.meta.client.upload_file(cfg_file, cfg['BUCKET'], 'config.yml')
log("Complete, instances will deploy next time pull-deploy.py is run with the --pull flag"
)
def pull(cfg_file):
cfg = load_config.get_config(cfg_file)
response = requests.get(
'http://169.254.169.254/latest/meta-data/instance-id')
instance_id = response.text
if (not instance_id):
log("Error: can't find instance ID from metadata server; are you sure you're running this on EC2?"
)
log('Starting deploy agent on instance ' + instance_id)
try:
result = run(instance_id, cfg)
except BaseException as e:
log('Error: ' + str(e))
result = False
if not result:
send_sns_log(
'Deploy ERROR for "' + cfg['NICKNAME'] + '" to "' + cfg['DOMAIN'] +
'": instance ' + instance_id, cfg['SNS_ERROR'])
if result == 2:
send_sns_log(
'Deploy COMPLETE for "' + cfg['NICKNAME'] + '" to "' +
cfg['DOMAIN'] + '": instance ' + instance_id, cfg['SNS_ERROR'])
def show(cfg_file):
log("Showing config from path: " + cfg_file)
cfg = load_config.get_config(cfg_file)
print(cfg)
This script exctact images from train folder and transfer them to validation folder
'''
def move_painting_to_validation(genre, name):
old = "./train/" + genre + "/" + name
new_dir = "./validation/" + genre
new = join(new_dir, name)
if not os.path.isdir(new_dir):
os.mkdir(new_dir)
os.rename(old, new)
config = load_config.get_config()
percencate_of_validation = float(config['validation']) #0.25
path_to_folder = config['path']
train_dir = path_to_folder + '/train'
os.chdir(path_to_folder)
genres = [f for f in listdir(train_dir) if isdir(join(train_dir, f))]
for genre in genres:
genre_path = join(train_dir, genre)
paintings = [f for f in listdir(genre_path) if isfile(join(genre_path, f))]
paintings_to_transfer = int(percencate_of_validation * len(paintings))
import os
from stat import S_ISREG
from task_scheduler import scheduled
import requests
import logging.config
from copy import deepcopy
from flask_api import status
from load_config import get_config
cfg = get_config()
conf = cfg['logging']
logging.config.dictConfig(conf)
log = logging.getLogger(__name__)
class FileWatcher:
def __init__(self,
pmmls_path,
extension='.pmml',
load_url='http://localhost:9001/load',
delete_url='http://localhost:9001/unload',
health_url='http://localhost:9001/health',
poll_delay_change=2,
poll_delay_app_service=2):
self.pmml_path = pmmls_path
self.extension = extension
self.load_url = load_url
self.delete_url = delete_url
self.health_url = health_url
self.files_metadata = {}
self.poll_delay_change = poll_delay_change