def inspect(self, context: Context) -> PropertyValues:
""" get properties by calling on the handler """
xsd = getns('xsd')
if self.handler is None:
#handler_factory = context['handler_factory']
handler_args = context['handler_args'] # dict of kwargs
logging.debug("in inspect, cntext has: {0}".format(context))
logging.debug(
"in inspect, handler_args has: {0}".format(handler_args))
self.handler = handlers.factory(**handler_args)
#self.handler = handler_factory(**handler_args)
retval = set()
predicate = context['predicate']
if predicate == 'dct:temporal':
context.push(handler=self.handler, predicate=predicate)
t = Temporal()
t.add_to_graph(context)
context.pop(('handler', 'predicate'))
retval.add(t.uri)
elif predicate == 'dcat:byteSize':
value = self.handler.size
xsd = getns('xsd')
retval.add(Literal(value, datatype=xsd.integer))
elif predicate == 'logset:recordCount':
if self.handler.num_records is not None:
retval.add(
Literal(self.handler.num_records, datatype=xsd.integer))
elif predicate == 'logset:estRecordCount':
if self.handler.num_records is not None:
retval.add(
Literal(self.handler.num_records, datatype=xsd.integer))
else:
buf = list(self.handler.get_slice(limit=10))
avg = sum([len(entry) for entry in buf]) / len(buf)
guess = self.handler.size / avg
retval.add(Literal(guess, datatype=xsd.integer))
elif predicate == 'logset:estRecordsPerDay':
# get timespan from handler, convert to days, etc
# the handler stores the timespan just as text, so need to convert
t_earliest = dateutil.parser.parse(self.handler.t_earliest)
t_latest = dateutil.parser.parse(self.handler.t_latest)
days = (t_latest - t_earliest).total_seconds() / 86400
nrecs = self.get_one_value('logset:recordCount') or \
self.get_one_value('logset:estRecordCount')
nrecs = int(float(str(nrecs))) # sorry
nrecs_per_day = int(nrecs / days)
retval.add(Literal(nrecs_per_day, datatype=xsd.integer))
return retval
def login(request):
login_form = forms.LoginForm()
login_message = ''
if request.method == 'POST':
login_form = forms.LoginForm(data=request.POST)
if login_form.is_valid():
try:
username = PFUser.objects.get(
email=login_form.cleaned_data['email']).user.username
user = auth.authenticate(
username=username,
password=login_form.cleaned_data['password'])
if user is not None:
if user.is_active:
auth.login(request, user)
return HttpResponseRedirect(
request.GET.get('next') or '/')
else:
login_message = 'Account disabled'
else:
login_message = 'Wrong email or password'
except PFUser.DoesNotExist: #@UndefinedVariable
login_message = 'Wrong email or password'
context = {
'login_form': login_form,
'login_message': login_message,
'navigation_pos': 'home',
'next': request.GET.get('next') or '/',
}
return render_to_response('login.html', Context(context, request))
def beta_invitations(request):
if request.method == 'POST':
form = forms.InvitationForm(data=request.POST)
if form.is_valid():
new_invitation = BetaInvitation()
new_invitation.assigned_customer = form.cleaned_data[
'requesting_customer']
new_invitation.forced_package = Package.objects.get(
code=FORCED_PACKAGE_CODE)
new_invitation.save()
new_invitation.password = generate_onetimepass(new_invitation.id)
new_invitation.save()
return HttpResponseRedirect(reverse('beta_invitations'))
else:
form = forms.InvitationForm()
invitations = BetaInvitation.objects.all().order_by('-invitation_date')
context = {
'invitations': invitations,
'form': form,
}
return render_to_response('beta_invitations.html',
Context(context, request))
def beta_requests_list(request):
requests = BetaTestRequest.objects.all().order_by('-request_date')
context = {
'requests': requests,
}
return render_to_response('beta_requests.html', Context(context, request))
def control_panel(request):
workers = Worker.objects.all()
context = {
'workers': workers,
'navigation_pos': 'home',
}
return render_to_response('control_panel.html', Context(context, request))
def load_history(request, worker_id=None):
worker = Worker.objects.get(id=worker_id)
load_infos = WorkerLoadInfo.objects.extra(
where=['worker_id=%d' % (int(worker_id))], order_by=['timestamp'])
context = {
'worker': worker,
'load_infos': load_infos,
}
return render_to_response('load_history.html', Context(context, request))
def beta_invitation_export(request):
requests = BetaTestRequest.objects.all().order_by('-request_date')
invitations = BetaInvitation.objects.filter(
beta_requester__isnull=True).order_by('-invitation_date')
context = {
'requests': requests,
'invitations': invitations,
}
return render_to_response('beta_invitation_export.html',
Context(context, request))
def log_info(request):
class ddict(dict):
def __init__(self, default):
self.default = default
def __getitem__(self, key):
return self.setdefault(key, self.default())
ndict = lambda: ddict(lambda: 0)
ps = subprocess.Popen(
'tail -1000000 /data/logs/api.log | grep "Search:INFO"',
shell=True,
stdout=subprocess.PIPE)
sums = ddict(ndict)
sucs = ddict(ndict)
cnts = ddict(ndict)
for l in ps.stdout:
m = re.match(
r'.*(\d{2}/\d{2}-\d{2}.\d)\d.*\[(\d{3})] in (\d+\.\d{3})s for \[(.*) /v1/indexes/([^\]]*)/([^\]]*)\]',
l)
if m:
dt = m.group(1)
code = m.group(2)
time = float(m.group(3))
act = m.group(4)
cod = m.group(5)
met = m.group(6)
req = act + ' ' + met
sums[req][dt] += time
cnts[req][dt] += 1
if code == '200':
sucs[req][dt] += 1
count = 1
rows_by_req = {}
for req, v in sums.iteritems():
rows = []
rows_by_req[req] = rows
count += 1
for dt, time in v.iteritems():
y = 2010
m = int(dt[3:5]) - 1
d = int(dt[0:2])
h = int(dt[6:8])
i = int(dt[9:10]) * 10
avg = time / cnts[req][dt]
rows.append('[new Date(%d, %d, %d, %d, %d), %f]' %
(y, m, d, h, i, avg))
context = {'rows_by_req': rows_by_req}
return render_to_response('log_info.html', Context(context, request))
def worker_resource_map(request, worker_id):
w = Worker.objects.get(id=int(worker_id))
w.sorted_deploys = sorted(w.deploys.all(), key=_size, reverse=True)
xmx = 0
for d in w.sorted_deploys:
xmx += d.effective_xmx
w.xmx = xmx
w.used = w.get_used_ram()
context = {'worker': w}
return render_to_response('worker_resource_map.html',
Context(context, request))
def add_to_graph(self, context: Context = None):
if self._in_graph:
logging.debug("already in graph, skipping")
return
if context is None:
context = Context()
# describe my properties first, so subclasses can use them to generate a helpful uri if necessary:
# I think that to avoid loops we need to do all of the asking (the user)
# before doing any of the adding to graph:
triples = []
context.push(node=self)
for predicate in self.properties:
context.push(predicate=predicate)
# need to convert string eg foaf:name to an actual uri for adding
# to graph:
logging.debug("calling Graph.geturi on {0}".format(predicate))
pred_uri = graph.geturi(predicate)
logging.debug("calling get_values with {0}, {1}".format(
str(predicate), str(context)))
for v in self.get_values(predicate, context):
if isinstance(v, Identifier):
triples.append((self.uri, pred_uri, v))
elif isinstance(v, Node):
triples.append((self.uri, pred_uri, v.uri))
else:
# I'm pretty sure this should never happen
raise Exception("oh oh! " + str(v) + " ... " +
str(type(v)))
context.pop(('predicate', ))
context.pop(('node', ))
for triple in triples:
logging.debug("adding triple {0}".format(triple))
self.graph.add(triple)
# finally, describe me:
rdf = graph.getns('rdf')
myclass = graph.geturi(self.rdf_class)
logging.info("adding me to graph: {0}, {1}, {2}".format(
self.uri, str(rdf.type), str(myclass)))
self.graph.add((self.uri, rdf.type, myclass))
self._in_graph = True
def index_history(request, worker_id=None, index_id=None):
index = Index.objects.get(id=index_id)
worker = Worker.objects.get(id=worker_id)
index_infos = WorkerIndexInfo.objects.filter(worker__id=worker_id,
deploy__index__id=index_id)
context = {
'index': index,
'worker': worker,
'index_infos': index_infos,
}
return render_to_response('index_history.html', Context(context, request))
def resource_map(request):
if request.method == 'POST':
if request.POST['task'] == 'redeploy':
id = request.POST['index_id']
rpc.get_deploy_manager().redeploy_index(
Index.objects.get(pk=id).code)
return HttpResponseRedirect('/resource_map')
workers = [
w for w in Worker.objects.select_related(depth=5).order_by('id').all()
]
for w in workers:
w.sorted_deploys = sorted(w.deploys.all(), key=_size, reverse=True)
w.used = w.get_used_ram()
context = {'workers': workers, 'packages': Package.objects.all()}
return render_to_response('resource_map.html', Context(context, request))
def manage_worker(request, worker_id=None):
context = {}
worker = Worker.objects.get(id=worker_id)
mount_infos = WorkerMountInfo.objects.extra(where=[
'worker_id=%d and timestamp=(select max(timestamp) from %s where worker_id=%d)'
% (int(worker_id), WorkerMountInfo._meta.db_table, int(worker_id))
],
order_by=['-used'])
indexes_infos = WorkerIndexInfo.objects.extra(where=[
'worker_id=%d and timestamp=(select max(timestamp) from %s where worker_id=%d)'
% (int(worker_id), WorkerIndexInfo._meta.db_table, int(worker_id))
],
order_by=['-used_mem'])
load_info = WorkerLoadInfo.objects.extra(where=[
'worker_id=%d and timestamp=(select max(timestamp) from %s where worker_id=%d)'
% (int(worker_id), WorkerLoadInfo._meta.db_table, int(worker_id))
])[0]
used_disk_total = 0
used_mem_total = 0
mount_percentages = {}
for info in mount_infos:
mount_percentages[info.mount] = (float)(
info.used) / (info.used + info.available) * 100
for info in indexes_infos:
used_disk_total += info.used_disk
used_mem_total += info.used_mem
context = {
'mount_infos': mount_infos,
'mount_percentages': mount_percentages,
'indexes_infos': indexes_infos,
'load_info': load_info,
'used_disk_total': used_disk_total,
'used_mem_total': used_mem_total,
'worker': worker,
'navigation_pos': 'home',
}
return render_to_response('manage_worker.html', Context(context, request))
def get_values(self,
predicate: str,
context: Optional[Context] = None) -> PropertyValues:
""" return a set of values for a property """
if context is None:
context = Context(predicate=predicate)
logging.debug("looking for {0} in {1}".format(predicate,
str(self.properties)))
props = self.properties.get(predicate)
if len(props) == 0:
logging.debug("calling a getter for {0}".format(predicate))
getter = getattr(self, self.getters[predicate])
logging.debug("got getter {0}, context {1}".format(
getter, context))
generator = (v for v in getter(context))
self.properties.add(predicate, *generator)
logging.debug("now {0} has: {1}".format(
predicate, str(self.properties[predicate])))
return self.properties[predicate]
def mount_history(request, worker_id=None, mount=None):
worker = Worker.objects.get(id=worker_id)
if not mount.startswith('/'):
mount = '/' + mount
mount_infos = WorkerMountInfo.objects.extra(
where=['worker_id=%d and mount="%s"' % (int(worker_id), mount)],
order_by=['timestamp'])
mount_percentages = {}
for info in mount_infos:
mount_percentages[info.timestamp] = (float)(
info.used) / (info.used + info.available) * 100
context = {
'worker': worker,
'mount': mount,
'mount_percentages': mount_percentages,
'mount_infos': mount_infos,
}
return render_to_response('mount_history.html', Context(context, request))
def execute(self, args):
topdir = args.dir[0]
# namespace should have at least 1 '/' and should end in '#'.
# the label for the LogSet itself will be the text between
# the last '/' and the '#', and the LogSet will be written to
# a file called {label}.ttl:
nstart = args.namespace[0].rfind('/') + 1
nend = -1 if args.namespace[0][-1] == '#' else None
base = args.namespace[0][:nstart]
name = args.namespace[0][nstart:nend]
ns = rdflib.Namespace(base + name + '#')
uri = ns[name]
#logging.info("logset uri is {0}".format(uri))
catalog = args.catalog[0] if args.catalog else None
fname = '{0}.ttl'.format(name)
if os.path.exists(fname):
msg = "{0} already exists, please move it or request ".format(
fname)
msg += "a different namespace and try again"
raise Exception(msg)
# flatten additional urls into a single list:
urls = [u for urllist in args.url for u in urllist]
#urls = args.url[0] or []
logging.debug("got args: {0}".format(str(args)))
logging.debug("got urls: {0}".format(str(urls)))
g = graph.construct(*urls, spider=True)
#print("the graph is now: {0}".format(graph.Graph.the_graph))
newindex = LogSet(uri=uri)
logging.debug("created a Logset newindex: {0}".format(str(newindex)))
newindex.add_to_graph()
# ask user what (high-level) subjects the logs are about, to help in
# guessing subjects of concretelogs
subjects = set()
prompt = "Please indicate the (high-level) subjects of these logs, or some (n)ew ones: "
while len(subjects) == 0:
context = Context(prompt=prompt)
subjects = Subject.select_from_known(context)
context.pop(('prompt', ))
prompt = "please indicate at least 1 subject!"
guess = newindex.get_one_value(
'dcat:landingPage') or "http://example.com"
guess += "/logs.tar.gz"
prompt = "What URL can these logs be accessed via? \n"
prompt += "(eg, " + guess + "\n"
# FIXME: should this actually be a URIRef?
accessURL = Literal(UI.ask(prompt, guess))
# new context for cataloging the actual data sources:
context = Context(logset=newindex,
subjects=subjects,
namespace=newindex.namespace)
context.push({'dcat:accessURL': accessURL})
datasource = DataSource.factory('ddict:files')
datasource.catalog(topdir, context)
## series to search for logs of:
#todo = set(LogSeries.known())
#done = set()
#logging.debug("todo has: {0}".format([', '.join(str(i)) for i in todo])))
## move this into the logformattype:
## files to finding matching series for:
#baselen = len(topdir)+1
#remaining = set([ FileInfo(topdir, path[baselen:], f)
# for path, d, files in os.walk(topdir) for f in files ])
#while len(todo) > 0:
# logseries = todo.pop()
# remaining = logseries.catalog(remaining, context)
# done.add(pattern)
# todo.remove(pattern)
# if len(todo)==0:
# # we need to come up with more logseries
# break # for now
# write the new index:
subgraph = graph.subgraph(newindex.prefix)
with open(fname, 'w') as f:
f.write(subgraph.serialize(format='n3').decode('ascii'))
# TODO make this less fragile
# write any new entities and localdict too:
try:
subgraph = graph.subgraph('new_entities')
with open('new_entities.ttl', 'w') as f:
f.write(subgraph.serialize(format='n3').decode('ascii'))
except KeyError:
# no new entities to write
pass
try:
subgraph = graph.subgraph('localdict')
with open('localdict.ttl', 'w') as f:
f.write(subgraph.serialize(format='n3').decode('ascii'))
except KeyError:
# no new entities to write
pass
import os
from torch.utils.data import Dataset
import json
from PIL import Image
from util import Context, parse_args
import random
import numpy as np
from torchvision import transforms as transforms
args = parse_args()
Context().init_by_args(args)
config = Context().get_config()
logger = Context().get_logger()
DEBUG = Context().DEBUG
CROP_SIZE = config['transform']['patch_size']
CHANNEL_NUM = config['transform']['channel_num']
def isHorizontalFlip(p=0.5):
if random.random() < p:
return True
else:
return False
def randomCropper(img_size, patch_size=CROP_SIZE):
h, w = img_size
p_h, p_w = patch_size
random_range_h = h - p_h
import copy
import time
import torch
from scipy.stats import spearmanr, pearsonr
from dataset import ImageDataset as dataset
from NestedNet import Model as model
from util import Context
from sklearn.metrics import mean_squared_error
from math import sqrt
config = Context().get_config()
logger = Context().get_logger()
# model
MODEL_SAVE_PATH = config["project"]["save_model"]
MODEL_LOAD_PATH = config["project"]["load_model"]
# datasetScores
INPUT_PATH = config["dataset"]["image_dir"]
LIST_SCORE = config["dataset"]["score_file"]
# train para
LEARNING_RATE = config['train']['learning_rate']
NUM_EPOCHS = config['train']["num_epochs"]
MUTI_GPU_MODE = config["train"]["muti_gpu"]
WEIGHT_DECAY = config["train"]["weight_decay"]
def train_model(model,
device,
optimizer,
def catalog(self, topdir: Set[FileInfo],
context: Context) -> Set[FileInfo]:
baselen = len(topdir) + 1
remaining = set([
FileInfo(topdir, path[baselen:], f)
for path, d, files in os.walk(topdir) for f in files
])
# keep track of the known relevant logseries to compare samples
# of new filepatterns against
#known_logseries = set()
# for a Files datasource, only file-based log formats make sense:
for fmttype in FileBasedLogFormatType.known(
{'logset:dataSource': self.uri}):
logging.info("found logformattype {0}".format(fmttype.uri))
context.push(logFormatType=str(fmttype.uri))
#fmttype_uri = graph.Graph.geturi(fmttype.uri)
for logseries in LogSeries.known(
{'logset:logFormatType': fmttype.uri}):
logging.info("found logseries {0} {1}".format(
fmttype.uri, str(logseries)))
#known_logseries.add(logseries)
# get logseries to narrow the subjects a bit
# then push the narrowed subject list to context
subjects = logseries.identify_subjects(context['subjects'])
context.push(filepatterns=logseries.filePatterns,
formatinfo=logseries.fmtInfo,
subjects=subjects,
logseries_uri=logseries.uri)
remaining = fmttype.catalog(remaining, context)
context.pop(('filepatterns', 'formatinfo', 'subjects'))
context.pop(('logFormatType', ))
while len(remaining) > 0:
# try describing new logseries for them:
logging.warning("{0:d} files remain to be catalogued!".format(
len(remaining)))
sample = remaining.pop()
break # code below is incomplete, jump out for now
# things to check:
# - is the mediatype suitable for a fmttype?
# (need to call on the fmttype for that I think, because
# eg FilePerTimePoint has inode mediatype, but the sample will
# be a file, so we'd be looking for the directory rather
# that the file itself
# - does the filename have something in common with a
# known logseries (of this fmttype)
# (for each known pattern, split the filename by that pattern
# and compare each non-matching part with the filepatterns
# of this logseries
# "clues" - parts of filename not matching patterns
# (what if 2 parts of the filename match a pattern?)
# if we get a clue called, eg "-" then it is probably not a
# useful one .. so skip too-short clues (maybe that don't have
# [a-z]+ in them
# ah, then from known logseries, split tags out and look for [a-z]+
# then propose a logseries if it has a matching [a-z]+ word
clues = set()
too_short = 2 # only consider words longer than this
re_word = re.compile('[a-zA-Z_]+')
for p in Pattern.known():
tag = p.properties.one('logset:tag')
regex = re.compile(p.properties.one('logset:regex'))
parts = regex.split(sample.filename)
# parts has the non-matching parts of the filename
if len(parts) == 1:
continue # no match
for part in parts:
#find just the words
# this will automagically handle <label:> patterns too
for w in re_word.findall(part):
if len(w) > too_short:
clues.add(w)
re_tag = re.compile('(<[\w:]+>)')
# TODO might be worth caching:
for fmttype in FileBasedLogFormatType.known(
{'logset:dataSource': self.uri}):
if not fmttype.right_mime_type(sample):
continue
for logseries in LogSeries.known(
{'logset:logFormatType': fmttype.uri}):
# TODO I think this belongs as a method of LogSeries:
# eg if logseries.describes(sample): logseries.add_filepattern ; break
for p in logseries.fmtInfo['filepattern']:
parts = re_tag.split(p)
for w in re_word.findall(part):
if w in clues:
# this might be a new filepattern for this logseries,
# show the user a sample and ask them to verify
# TODO: make logset:sampleContents a property of LogSeries
# and show it to the user
logging.info("{0} might be a {1}".format(
str(sample), str(logseries)))
# TODO break out of inner loops if the candidate is found
else:
pass
# ask the user if this file looks like this logformattype (eg
# "is this a timestampedlogfile?"
# might be a function of the handler, to ask user if this is right
# thing, and verify that it can read the fields it requires, eg the
# timestamp and component word
# having found or created a logsereis, now find other candidates that match
# it and catalog them
return remaining
def select_from_known(
cls,
context: Context = Context(),
label=None,
multi=True,
allow_create=True,
filters: Dict[str, str] = dict()) -> PropertyValues:
""" classmethod select getter ..."""
retval = set()
predicate = context.get('predicate')
target_cls = cls.targets.get(predicate, cls)
known = list(target_cls.known(filters))
if label is None:
label = target_cls.__name__
if multi:
default_prompt = "Please select one or more {0} "
default_prompt += "(space-separated{1} or empty when done) "
#ui_method = UI.multi_select
else:
default_prompt = "Please select a {0}{1}"
#ui_method = UI.select
prompt_new = ", or (n)ew " if allow_create else ""
additional = ['n'] if allow_create else []
prompt = (context.get('prompt') or cls.prompts.get(predicate)
or default_prompt).format(label, prompt_new)
if multi:
selection = UI.multi_select(prompt, known, *additional)
else:
selection = [UI.select(prompt, known, *additional)]
while True:
# loop so user can create multiple new entries
if len(selection) == 0:
break
# for multi, but with single it is same logic:
for choice in selection:
if allow_create and str(choice).lower() == 'n':
# now it gets giddyingly recursive, make a new target_cls Node
# and add that:
classname = target_cls.__name__
label = UI.ask(
"please give the new {0} a label: ".format(classname))
props = {target_cls.label_property: [label]}
new = target_cls(properties=props)
new.add_to_graph(context)
uri = new.uri
else:
logging.info(
f"got choice {choice} from selection {selection}")
#obj = known[choice]
obj = choice
uri = obj.uri
retval.add(uri)
selection = UI.multi_select("more? ", known, 'n') if multi else []
#selection = ui_method("more? ", known, 'n') if multi else []
return retval
def catalog(self, candidates: Set[FileInfo],
context: Context) -> Set[FileInfo]:
filepatterns = context['filepatterns']
matching = set()
for regex in filepatterns:
logging.info("filepattern: {0}: {1}".format(regex, regex.pattern))
logging.info("{0:d} candidates".format(len(candidates)))
logging.info("candidates: {0}".format(candidates))
logging.info(
"after filtering: " +
str(filter(lambda x: regex.match(x.filename), candidates)))
matching |= set(
filter(lambda x: regex.match(x.filename), candidates))
# filter by MIMEtype:
mediatypes = self.properties['dcat:mediaType']
mimetest = lambda x: self.right_mime_type(x)
matching -= set(filter(mimetest, matching))
# args for the LogFormatType that will handle the actual file/source:
handler_args = {
'rdf_class': context['logFormatType'],
'fmtinfo': context['formatinfo']
}
# 'properties': self.properties
#context.push(handler_factory=handlers.factory)
context.push(handler_args=handler_args)
# hmm, everything the ConcreteLog infers, we can just pass as properties:
common_properties = MultiDict()
common_properties.add('logset:isInstanceOf', context['logseries_uri'])
common_properties.add('dcat:accessURL', context['dcat:accessURL'])
#common_properties.add_values('logset:subject', context['subjects'])
common_properties.add('logset:subject', *context['subjects'])
common_properties.add('namespace', context['namespace'])
logging.info("logset for concretelog has: {0}".format(
context['logset']))
# in most cases we are looking file-by-file .. FilePerTimepoint
# can override this default behavior
for f in matching:
#context.push(label=f.filename)
#context.push({'dcat:downloadURL': f.relpath + os.sep + f.filename})
properties = MultiDict(common_properties)
properties.add('rdfs:label', Literal(f.filename))
relpath = (f.relpath + os.sep + f.filename).lstrip(os.sep)
properties.add('dcat:downloadURL', Literal(relpath))
logging.info(
"properties for concretelog has: {0}".format(properties))
handler_args['target_url'] = os.sep.join(f) # full local path
log = ConcreteLog(properties=properties)
logging.info("adding log to graph: {0}".format(log))
try:
log.add_to_graph(context)
except UnsupportedLogFormatHandler as err:
logging.warn("logformat {0} not implemented".format(err))
properties.remove('rdfs:label')
properties.remove('dcat:downloadURL')
context.pop(('handler_args', ))
#context.pop('handler_factory')
return candidates - matching
from util import Context
from basic_block import *
from pytorch_ssim import SSIM
from torch.nn import init
import math
import torch.utils.model_zoo as model_zoo
config = Context().get_config()
logger = Context().get_logger()
# model
CROP_SIZE = config['transform']['patch_size']
model_urls = {
'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
}
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
self.bn1 = nn.BatchNorm2d(planes)
self.conv2 = nn.Conv2d(planes,
planes,
kernel_size=3,
def biz_stats(request):
context = {}
return render_to_response('biz_stats.html', Context(context, request))
def operations(request):
#if request.method == 'POST':
# if request.POST['task'] == 'redeploy':
# id = request.POST['index_id']
# rpc.get_deploy_manager().redeploy_index(Index.objects.get(pk=id).code)
# return HttpResponseRedirect('/resource_map')
level = request.GET.get('level', 'top')
if level == 'top':
return render_to_response('operations/index.html', Context({},
request))
elif level == 'refresh':
data = {
'Config':
map(configuration_dict, IndexConfiguration.objects.all()),
'Account':
map(account_dict,
Account.objects.select_related('user').all()),
'Deploy':
map(deploy_dict, Deploy.objects.all()),
'Index':
map(index_dict, Index.objects.all()),
'Package':
map(package_dict, Package.objects.all()),
'Worker':
map(worker_dict, Worker.objects.all()),
}
return JsonResponse(data)
elif level == 'index':
id = request.GET.get('id')
index = Index.objects.get(pk=id)
data = {
'Index': index_dict(index),
'Deploy': map(deploy_dict, index.deploys.all()),
}
return JsonResponse(data)
elif level == 'stats':
id = request.GET.get('id')
d = Deploy.objects.get(pk=id)
client = rpc.getThriftIndexerClient(d.worker.lan_dns, int(d.base_port),
3000)
return JsonResponse(client.get_stats())
elif level == 'log':
id = request.GET.get('id')
file = request.GET.get('file')
d = Deploy.objects.get(pk=id)
client = rpc.get_worker_controller(d.worker, 4000)
lines = client.tail(file, 300, d.index.code, d.base_port)
return JsonResponse(lines)
elif level == 'redeploy':
id = request.GET.get('id')
rpc.get_deploy_manager().redeploy_index(Index.objects.get(pk=id).code)
return HttpResponse()
elif level == 'decommission':
id = request.GET.get('id')
Worker.objects.filter(id=id).update(
status=Worker.States.decommissioning)
return JsonResponse(worker_dict(Worker.objects.get(id=id)))
elif level == 'delete_worker':
id = request.GET.get('id')
w = Worker.objects.get(id=id)
if w.status != Worker.States.decommissioning:
return HttpResponse('worker not decommissioning', status=409)
if w.deploys.count():
return HttpResponse('worker not empty', status=409)
w.delete()
return HttpResponse()
elif level == 'delete_account':
id = request.GET.get('id')
a = Account.objects.get(id=id)
user = a.user.user
if a.indexes.count():
return HttpResponse('account has index', status=409)
if a.payment_informations.count():
return HttpResponse('account has payment information', status=409)
user = a.user.user
a.delete()
user.delete()
return HttpResponse()
elif level == 'account_set_pkg':
id = request.GET.get('id')
pid = request.GET.get('pkg')
p = Package.objects.get(id=pid)
updated = Account.objects.filter(id=id).update(package=p)
if updated:
return JsonResponse(account_dict(Account.objects.get(id=id)))
else:
return HttpResponse('account not found', status=409)
elif level == 'account_set_cfg':
id = request.GET.get('id')
cid = request.GET.get('cfg')
c = IndexConfiguration.objects.get(id=cid)
updated = Account.objects.filter(id=id).update(configuration=c)
if updated:
return JsonResponse(account_dict(Account.objects.get(id=id)))
else:
return HttpResponse('account not found', status=409)
elif level == 'index_set_cfg':
id = request.GET.get('id')
cid = request.GET.get('cfg')
c = IndexConfiguration.objects.get(id=cid)
updated = Index.objects.filter(id=id).update(configuration=c)
if updated:
return JsonResponse(index_dict(Index.objects.get(id=id)))
else:
return HttpResponse('index not found', status=409)
return HttpResponseNotFound()
def accounts_info(request):
context = {
'accounts': Account.objects.all().order_by('package', 'creation_time')
}
return render_to_response('accounts_info.html', Context(context, request))