def post(self, action):
res = Res()
if action == "sync":
try:
s = json.loads(self.request.body)
db_name = s['database']
del s['database']
except ValueError as e:
print e
self.finish(res.emit())
return
except KeyError as e:
print e
self.finish(res.emit())
return
db = None
if db_name == "M2X":
from ISData.m2xdb import M2XDB
db = M2XDB()
if db is not None:
db.updateConfig(s)
res.result = STATUS_OK[0]
else:
activate = False
if action == "start":
activate = True
res.data = {
"started" : [],
"stopped" : []
}
for scraper in getScrapers(scraper_dir):
s = Schema(scraper['url'])
if s.is_active != activate:
status = "started"
if not activate:
status = "stopped"
res.data[status].append(s._id)
s.activate(activate=activate)
res.result = STATUS_OK[0]
self.finish(res.emit())
def post(self): res = Res() try: s = json.loads(self.request.body) except ValueError as e: print e self.finish(res.emit()) return id_ = s['id'] del s['id'] print "update config" f = open(os.path.join(scraper_dir, id_, "conf.json"), 'rb') schema = Schema(json.loads(f.read())['url']) f.close() should_activate = None for key in s.keys(): if key == "is_active": should_activate = s[key] continue if type(s[key]) == dict: val = getattr(schema, key) for key_ in s[key].keys(): val[key_] = s[key][key_] schema.setattr(key, val) else: schema.setattr(key, s[key]) schema.save() if should_activate is not None: print "with activation: %s!" % should_activate schema.activate(activate=should_activate) res.result = STATUS_OK[0] print res.emit() self.finish(res.emit())
def post(self): res = Res() if not self.validateRequest(): self.finish(res.emit()) return try: s = json.loads(self.request.body)['manifest'] except ValueError as e: print e self.finish(res.emit()) return url = s['url'] del s['url'] as_test = False try: as_test = s['confirm'] print "found confirm directive: %s" % as_test del s['confirm'] except KeyError as e: print "no confirm directive" print e pass as_test = True if not as_test: schema = Schema(url, create=True, **s) schema.save() self.finish(schema.activate()) else: schema = Schema(url, create=False, as_test=True, **s) print json.dumps(schema.emit()) self.finish(schema.scrape(as_test=True))