def do_generate_packages(mid):
try:
print 'DEBUG going to the background maker...'
ret = {}
ret['code'] = '50'
if mid != -1:
# TODO - will use my_qiniutoken() to pass a token to consumer
tok = __my_qiniutoken()
handler = createZipBall.delay(tok, mid)
print 'DEBUG waiting...'
handler.wait()
# Note - consumer download,package, and upload, give a MD5
# after wait(), here will try generate a uuid-like pincode
# to mark the package
st, fsize, md5 = handler.get()
print '-the MQ handling result is: %s|%s ' % (st, md5)
if st == 200:
# try update the package data
obj = Packages.objects.get(id=mid)
# 2015-09-10 Use random 6-digit code
#code = __my_createpincode()
code = __my_createpincode_numerical()
# for the new package, pincode need be a new created one
if obj.md5 is None:
obj.pincode = code
ret['pinCode'] = code
else:
if obj.pincode is None:
obj.pincode = code
ret['pinCode'] = code
else:
ret['pinCode'] = obj.pincode
obj.md5 = md5
dt = datetime.datetime.now()
obj.completed = dt.strftime('%Y-%m-%d %H:%M:%S')
obj.size = fsize
obj.save()
ret['code'] = '0'
else:
print 'seems sth wrong(http status:%s)' % (st)
ret['code'] = st
else:
ret['code'] = '51' #incorrect request
except:
base_dir = utils.get_base_dir()
os.chdir(base_dir)
info = "%s || %s" % (sys.exc_info()[0], sys.exc_info()[1])
print 'an exception' + info
ret['code'] = '52' #incorrect request
return ret
def sleep(self, seconds, board_type_id):
path = os.path.abspath(get_base_dir() + "/board.json")
with open(path, 'r') as f:
board_json = json.load(f)
board = board_json[board_type_id - 1]
try:
yield self.api('{}/{}'.format(board['sleep_api'], seconds),
method="POST")
except Exception as e:
gen_log.error(e)
raise
def get_temp(self, board_type_id):
path = os.path.abspath(get_base_dir() + "/board.json")
with open(path, 'r') as f:
board_json = json.load(f)
board = board_json[board_type_id - 1]
try:
result = yield self.api(board['temp_api'],
method="GET",
request_timeout=5)
temp = result['celsius_degree']
except Exception as e:
gen_log.error(e)
raise
raise gen.Return(temp)
def get_config(self, ignore_specials = True):
"""
Returns config for this module
:param ignore_specials: Omits special keys
:return ConfigParser section
"""
# Load config
base = get_base_dir()
config = ConfigParser()
config.read(base + sep + CONFIG_FILENAME)
section = config.items(self.name)
if ignore_specials:
for data in tuple(section):
if data[0] in self.specials:
section.remove(data)
return section
def load_model_params(model, debug=False):
tf.reset_default_graph()
model_path = '%s/trained_models/%s' % (utils.get_base_dir(), model)
with open('%s/network-params.json' % model_path) as jsonfile:
model_params = json.load(jsonfile)
model_params['model_path'] = '%s/model' % model_path
keys = list(model_params.keys())
keys.sort()
table = PrettyTable()
table.field_names = ["Parameter", "Value"]
for k in keys:
table.add_row([k, model_params[k]])
table.align = 'l'
if debug:
print(model_path)
print('Load %s model' % model)
print(table)
# sess = tf.session()
# params_dict = {
# 'n_bumps': model_params['n_bumps'],
# 'dim' : model_params['dim'],
# 'n_hidden': model_params['n_hidden'],
# 'forget_bias': model_params['forget_bias'],
# 'n_steps': model_params['n_steps'],
# 'l': model_params['gp_length'],
# 'scope': model_params['scope']
# }
# build_training_graph(**params_dict)
# saver = tf.train.Saver()
# saver.restore(sess, '%s/model' % model_path)
return model_params
def add_ota(self, board_type_id):
path = os.path.abspath(get_base_dir() + "/board.json")
with open(path, 'r') as f:
board_json = json.load(f)
board = board_json[board_type_id - 1]
try:
body = board['config']
result = yield self.api(
'/v1/ota/trigger',
body=body,
method="POST",
headers={"Content-Type": "application/json"})
except Exception as e:
gen_log.error(e)
raise
raise gen.Return({
"status": result['ota_status'],
"status_text": result['ota_msg']
})
def parse_init(required=True):
base_template_dir = os.path.join(utils.get_base_dir(__file__),
TEMPLATE_DIRECTORY)
available_templates = [
item for item in os.listdir(base_template_dir)
if os.path.isdir(os.path.join(base_template_dir, item))
]
parser = argparse.ArgumentParser(
description='Create new system based on a template')
parser.add_argument('--template',
action="store",
choices=available_templates,
required=required)
parser.add_argument('--action',
action="store",
choices=ACTIONS,
required=True)
parser.add_argument('--env',
action="store",
choices=ENVS,
required=required)
parser.add_argument('--flag',
action="append",
help="Flags to be applied. Only steps with included "
"flags will be executed")
parser.add_argument(
'--var',
action="append",
help="Override default project vars. Each allowed var "
"is defined in its environment documentation. Format -> "
"varname:varvalue")
parser.add_argument(
'--debug',
action="store_true",
help="Add this argument to get extra verbosity on each "
"command")
if required:
parser.add_argument('projectname', action="store")
else:
parser.add_argument('projectname', action="store", nargs='?')
return parser.parse_args()
def setup_loggers():
logdir = os.path.join(get_base_dir(), 'log')
safe_mkdir(logdir)
def setup_file_logger(logger):
short_name = logger.name[len(LOGGER_NAME_PREFIX):]
file = os.path.join(logdir, '{name}.log'.format(name=short_name))
fileHandler = logging.FileHandler(file)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
fileHandler.setFormatter(formatter)
logger.addHandler(fileHandler)
setup_file_logger(app.logger)
# set logger class for future loggers
class RequestLogger(logging.Logger):
def __init__(self, name):
super(RequestLogger, self).__init__(name)
setup_file_logger(self)
self.setLevel(logging.INFO)
logging.setLoggerClass(RequestLogger)
''' Configuration file for the program '''
import time
import json
from types import SimpleNamespace
from urllib.parse import urljoin
import requests
from server import SERVER_URL, SERVER_AUTH
from utils import get_base_dir
BASE_DIR = get_base_dir()
def get_settings(logger, debug=False):
''' Parses the settings locally or from the server according to the debug value '''
if debug:
with open('settings.json') as file:
settings_ = json.load(file)
else:
while True:
try:
logger.log('Parsing settings from the server')
settings_ = requests.get(urljoin(SERVER_URL, 'api/v1/settings/'),
auth=SERVER_AUTH, timeout=30).json()
break
except requests.exceptions.RequestException:
time.sleep(10)
def setUp(self):
NER.init(
os.path.join(get_base_dir(), 'resources/models/ner/spacy_model'))
import os import pickle import time from tensorflow.python.keras import Model from tensorflow.python.keras.applications import InceptionV3 from tqdm import tqdm from utils import load_flicker8k_token, get_base_dir, clean_descriptions from utils import get_all_vocab, save_descriptions from utils import load_image_names, get_image_paths, load_clean_descriptions from extract_image_feature import image_preprocess from extract_image_feature import encode base_dir = get_base_dir() text_file_base_dir = os.path.join(base_dir, "data/captioning/TextFiles/") filename = os.path.join(text_file_base_dir, "Flickr8k.token.txt") # parse descriptions descriptions = load_flicker8k_token(filename) # clean descriptions cleaned_descriptions = clean_descriptions(descriptions) # summarize vocabulary vocabulary = get_all_vocab(descriptions) save_descriptions(descriptions, 'descriptions.txt') ##################################################################
def main():
modules = []
log = get_logger("Main", [stdout])
msg = "Module '%s' %s() returned code: %s"
try:
# Load modules
for filename in listdir(get_base_dir() + sep + MODULES):
if "__" in filename:
continue
name = filename.split(".py")[0]
try:
# Import file in dir
mod = importlib.import_module(MODULES + "." + name)
class_name = default_class(name)
cls = getattr(mod, class_name, None)
if cls is None:
log.error("Module '%s' doesn't have class: %s" % (filename, class_name))
module = cls(modules)
# Store to loaded or unload it
if module.enable:
modules.append(module)
log.info("Module '%s' loaded from %s" % (module.name, filename))
else:
code = module.unload()
log.info(
"Module '%s' was loaded from %s but is disabled, returned code: %s"
% (module.name, filename, code)
)
except ImportError as ie:
log.error("Module '%s' failed to import: %s" % (filename, ie))
raise ie
# Check if any loaded
total = len(modules)
if total == 0:
raise Exception("No module loaded!")
log.info("Loaded %s modules" % total)
# Run each module
while True:
for module in frozenset(modules):
# Run if its time
if module.is_interval():
code = module.run()
if code < 0:
raise Exception(msg % (module.name, "run", code))
log.debug(msg % (module.name, "run", code))
module.last_time = time()
# Remove if no interval
if module.interval == 0:
code = safe_call(module.unload, log, None, None, False)
log.debug(msg % (module.name, "unload", code))
sleep(0.5)
finally:
# Attempt to unload modules
for module in frozenset(modules):
code = safe_call(module.unload, log, None, None, False)
log.info(msg % (module.name, "unload", code))
def __init__(self):
jieba.lcut("你好")
self.base_dir = get_base_dir()
with open(os.path.join(self.base_dir, 'resources/data/中英文停用词.txt'), 'r', encoding='utf8') as f:
self.stopwords = [l.strip() for l in f.readlines()]
def get_logfile():
return '%s/log/assembly.txt' % utils.get_base_dir()
DEBUG = False
def parse_init():
parser = argparse.ArgumentParser(description='Create new template scaffold')
parser.add_argument('--debug', action="store_true",
help="Add this argument to get extra verbosity on each command")
parser.add_argument('templatename', action="store")
return parser.parse_args()
if __name__ == "__main__":
args = parse_init()
templatename = args.templatename
DEBUG = args.debug
base_dir = utils.get_base_dir(__file__)
template_dir = os.path.join(base_dir, "templates", templatename)
if os.path.exists(template_dir):
print("Template exists. Aborting")
exit(0)
for command in [
"mkdir -p %s" % template_dir,
"mkdir -p %s/dev" % template_dir,
"mkdir -p %s/prod" % template_dir,
"touch %s/README.md" % template_dir,
"touch %s/dev/README.md" % template_dir,
"touch %s/prod/README.md" % template_dir,
"touch %s/dev/docker-compose.yml.template" % template_dir,
"touch %s/prod/docker-compose.yml.template" % template_dir,
"touch %s/dev/defaultvars.json" % template_dir,
"touch %s/prod/defaultvars.json" % template_dir,
import re
from traceback import format_exc
from flask import Blueprint, request
from flask import current_app as app
from github import GitHubAPI
from utils import get_base_dir, get_config, dump_to_json
from app_log import log_request
app_issue = Blueprint('issue', __name__)
gitHub = GitHubAPI(base_dir=get_base_dir(), **get_config('github'))
escape_md_re = re.compile(r'([*_\\<>\[\]`])')
# stub for receiving events
@app_issue.route('/api/github-gate/', methods=['POST'])
def github_gate():
log_request(request, 'api-github-gate')
return 'ok'
def is_issue_request_valid(request):
exp_headers = {
'X-Requested-With': ('XMLHttpRequest',),
'Content-Type': (
'application/json; charset=utf-8',
'application/json; charset=UTF-8',
),
}
for k, exp_v in exp_headers.items():
value = request.headers.get(k)
def init_ner():
NER.init(os.path.join(get_base_dir(), 'resources/models/ner/spacy_model'))
