def wireless_post(): data = request.get_json(force=True) ldate = data["date"] ltime = data["time"] try: records = [ Wireless(source=item["source"], date=ldate, time=ltime, ap=item["AP"], mac=item["MAC"], lastip=item["Last IP"], dnsname=item["DNSname"], rx=item["RX"], tx=item["TX"], uptime=item["Uptime"], lastact=item["LastACT"], signalstrength=item["SignalStrength"], snr=item["SNR"], ccq=item["TX/RX-CCQ"], throughput=item["PThroughput"]) for item in data["items"] ] with database.atomic(): Wireless.bulk_create(records, batch_size=10) database.commit() except Exception as e: # make_dump(ldate, data, e) raise e return 'Ok'
def add_countries(): if config.QUERYAT_URL is None or config.GEOCODE_BATCH <= 0: return database.connect() with database.atomic(): q = Change.select().where( (Change.country >> None) & (Change.action != 'a') & (~Change.changes.startswith('[[null, null]'))).limit( config.GEOCODE_BATCH + 200) count = config.GEOCODE_BATCH for ch in q: coord = ch.changed_coord() if coord is not None: country = geocode(coord[0], coord[1]) if country is not None: ch.country = country[:150] ch.save() else: # print('Empty coordinates: {0} {1} {2}'.format(ch.id, ch.action, ch.changes.encode('utf-8'))) pass # We request more because of these empty coordinates errors count -= 1 if count <= 0: break
def changeset(): if 'osm_token' not in session: redirect(url_for('login')) cs_data = request.args.get('changeset') if not cs_data.strip(): return redirect(url_for('front')) user = get_user() # TODO: call submit_changeset instead try: changeset = parse_changeset_id(cs_data) cs_date, conforms = validate_changeset(user, changeset, None, openstreetmap) except ValueError as e: flash(str(e)) return redirect(url_for('front')) if not cs_date or cs_date != today(): flash('Date of the changeset is wrong') return redirect(url_for('front')) task = Task.get(Task.user == user, Task.day == cs_date) try: last_task = Task.select(Task.day).where( Task.user == user, Task.changeset.is_null(False)).order_by(Task.day.desc()).get() is_streak = last_task.day == cs_date - cs_date.resolution except Task.DoesNotExist: is_streak = False task.changeset = changeset task.correct = conforms if is_streak: user.streak += 1 else: user.streak = 1 user.score += int(math.log(user.streak + 1, 2)) if conforms: flash('An extra point for completing the task') user.score += 1 if user.level < len(config.LEVELS) + 1: if user.score >= config.LEVELS[user.level - 1]: user.level += 1 flash('Congratulations on gaining a level!') with database.atomic(): task.save() user.save() flash('Changeset noted, thank you!') return redirect(url_for('front'))
def submit_changeset(user, changeset, req=None): """Validates the changeset, records it and returns a series of messages.""" lang = load_language_from_user('', user)['validation'] try: changeset = parse_changeset_id(changeset) cs_date, conforms = validate_changeset(user, changeset, None, req) if not cs_date: raise ValidationError('wrong_date') last_task_day = get_last_task_day(user) if last_task_day and last_task_day >= cs_date: raise ValidationError('has_later_changeset') if cs_date < yesterday(): raise ValidationError('old_changeset') except ValidationError as e: return [e.to_lang(lang)], False task = Task.get(Task.user == user, Task.day == cs_date) task.changeset = changeset task.correct = conforms if last_task_day == cs_date - cs_date.resolution: user.streak += 1 else: user.streak = 1 user.score += int(math.log(user.streak + 1, 2)) msgs = [lang['changeset_noted'].format(user.streak)] if conforms: user.score += 1 msgs.append(lang['extra_point']) if user.level < len(config.LEVELS) + 1: if user.score >= config.LEVELS[user.level - 1]: user.level += 1 msgs.append(lang['gain_level']) with database.atomic(): task.save() user.save() return msgs, True
def process_notes(): database.connect() if not check_update(): return response = urllib2.urlopen(NOTES_URI) # Parsing bz2 through a temporary file tmpfile = TemporaryFile() while True: chunk = response.read(512*1024) if not chunk: break tmpfile.write(chunk) tmpfile.seek(0) with database.atomic(): with BZ2File(tmpfile) as f: for event, element in etree.iterparse(f): if element.tag == 'note': if len(element) > 0 and element[0].text and '#mapsme' in element[0].text: note_id = element.get('id') try: ch = Change.get(Change.changeset == note_id, Change.action == 'n') if element[-1].get('action') == 'closed' and ch.processed is None: print('Found closed note {0}'.format(note_id)) ch.processed = hour_difference(ch.timestamp, element[-1].get('timestamp')) ch.save() except Change.DoesNotExist: ch = Change() ch.action = 'n' ch.version = '' ch.changeset = note_id ch.user = element[0].get('user') if element[0].get('uid') else 'Anonymous Note' print('Found new note {0} by {1}'.format(note_id, ch.user.encode('utf-8'))) ch.timestamp = datetime.strptime(element[0].get('timestamp'), '%Y-%m-%dT%H:%M:%SZ') if element[-1].get('action') == 'closed' and ch.processed is None: ch.processed = hour_difference(ch.timestamp, element[-1].get('timestamp')) changes = [(element.get('lon'), element.get('lat')), {'note': element[0].text}] ch.changes = json.dumps(changes, ensure_ascii=False) ch.save() element.clear()
def add_countries(): if config.QUERYAT_URL is None or config.GEOCODE_BATCH <= 0: return database.connect() with database.atomic(): q = Change.select().where((Change.country >> None) & (Change.action != 'a') & (~Change.changes.startswith('[[null, null]'))).limit( config.GEOCODE_BATCH + 200) count = config.GEOCODE_BATCH for ch in q: coord = ch.changed_coord() if coord is not None: country = geocode(coord[0], coord[1]) if country is not None: ch.country = country[:150] ch.save() else: # print('Empty coordinates: {0} {1} {2}'.format(ch.id, ch.action, ch.changes.encode('utf-8'))) pass # We request more because of these empty coordinates errors count -= 1 if count <= 0: break
def insert_people(crd, data): people = data['people'] person_list = [] with database.atomic(): for person in people: row = mergedicts(getname(person['name']), adviser = crd) while True: try: entry = db.Person.get(**row); break except db.Person.DoesNotExist: db.Person.insert(**row).execute() percentage = re_PERCENTAGE.sub(r'.\1', person['ownership']) is_controlperson = person['controlperson'] db.Ownership.get_or_create(person = entry, percentowned = percentage, controlperson = is_controlperson) title = ' '.join(x.capitalize() for x in person['title'].split()) since = pd.to_datetime(person['since']) (db.Person.update(date = since, title = title) .where((db.Person.firstname == entry.firstname) & (db.Person.lastname == entry.lastname) & (db.Person.adviser == crd)).execute())
def parase_log(self): import re import datetime latest_log_offset = self.get_lastest_log_offset() matchRegex = r'^(.+) - tornado\.(\w+) - (\w+) - (\d+) (\w+) (.*?) \((.*?)\) (.*?)ms' queryTimeRegex = r'(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+),(\d+)' compileRegex = re.compile(matchRegex) timeCompiledRegex = re.compile(queryTimeRegex) with open(self.log_path, 'r+') as f: # move pointer to current offset f.seek(latest_log_offset, 0) data_source = [] for eachline in f.readlines(): info = compileRegex.findall(eachline) if len(info) == 1: # extract info from length timeString, logType, logLevel, requestStatus, requestType, requestURL, requestIP, requestDuration = info[ 0] # need to praser timestring first year, month, day, hour, minute, second, milsecond = timeCompiledRegex.findall( timeString)[0] reqDatetime = datetime.datetime(int(year), int(month), int(day), hour=int(hour), minute=int(minute), second=int(second), microsecond=int(milsecond)) thisLineLog = dict(queryTime=reqDatetime, logType=logType, logLevel=logLevel, requestStatus=requestStatus, requestType=requestType, requestURL=requestURL, requestIP=requestIP, requestDuration=requestDuration) data_source.append(thisLineLog) #print eachline cur_log_offset = f.tell() # print '# cur',cur_log_offset # store info BaseHandler.prepare(self) # bulk insert from db import database with database.atomic(): for idx in range(0, len(data_source), 100): log.insert_many(data_source[idx:idx + 100]).execute() lastest_log_offset = configOption.get(name='lastest_log_offset') lastest_log_offset.value = cur_log_offset lastest_log_offset.save() # save time import time lastest_log_update_time, created = configOption.get_or_create( name='lastest_log_update_time', defaults={'value': str(time.time())}) lastest_log_update_time.value = str(time.time()) lastest_log_update_time.save() BaseHandler.on_finish(self)
def upload_project(): def add_flash(pid, msg): flash(msg) return redirect(url_for('add_project', pid=pid)) user = get_user() if not is_admin(user): return redirect(url_for('front')) pid = request.form['pid'] if pid: pid = int(pid) project = Project.get(Project.id == pid) if not is_admin(user, project): return redirect(url_for('front')) update_audit(project) else: pid = None project = Project() project.feature_count = 0 project.bbox = '' project.owner = user project.name = request.form['name'].strip() if not project.name: return add_flash(pid, 'Empty name - bad') project.title = request.form['title'].strip() if not project.title: return add_flash(pid, 'Empty title - bad') project.url = request.form['url'].strip() if not project.url: project.url = None project.description = request.form['description'].strip() project.can_validate = request.form.get('validate') is not None project.validate_modified = request.form.get('validate_modified') is not None project.hidden = request.form.get('is_hidden') is not None if 'json' not in request.files or request.files['json'].filename == '': if not pid: return add_flash(pid, 'Would not create a project without features') features = [] else: try: features = json.load(codecs.getreader('utf-8')(request.files['json'])) except ValueError as e: return add_flash(pid, 'Error in the uploaded features file: {}'.format(e)) if 'features' not in features or not features['features']: return add_flash(pid, 'No features found in the JSON file') features = features['features'] audit = None if 'audit' in request.files and request.files['audit'].filename: try: audit = json.load(codecs.getreader('utf-8')(request.files['audit'])) except ValueError as e: return add_flash(pid, 'Error in the uploaded audit file: {}'.format(e)) if not audit: return add_flash(pid, 'No features found in the audit JSON file') proj_audit = json.loads(project.audit or '{}') if audit: proj_audit.update(audit) project.audit = json.dumps(proj_audit, ensure_ascii=False) if features or audit or not project.updated: project.updated = datetime.datetime.utcnow().date() project.save() if features: with database.atomic(): update_features(project, features, proj_audit) if project.feature_count == 0: project.delete_instance() return add_flash('Zero features in the JSON file') return redirect(url_for('project', name=project.name))
def startScan(self, *args, **kwargs): # we will use plagiarism system permission import probCrawler self._q = queues.Queue() # email = self.get_argument('email') self.query = probCrawler.crawler(queryName='') accountList = cronInfo.filter(isPermit=True)[:] accounts = set([cronAccount.account for cronAccount in accountList]) for cronAccount in accounts: print(cronAccount) self.genTask(name=cronAccount) yield self._q.join(timeout=timedelta(seconds=100)) print(self.infoDict) dataSource = [] # database storage for name, dataDict in self.infoDict.items(): # get total data tot = 0 submitTot = 0 for oj, resTuple in dataDict.items(): ac, submit = resTuple tot += int(ac) submitTot += int(submit) # change tuple str redirecting dataDict[oj] = (int(ac), int(submit)) if submitTot != 0: ratio = tot / submitTot else: ratio = 0 defaultOption = (0, 0) saveData = dict( name=name, pojNum=dataDict.get('poj', defaultOption)[0], hduNum=dataDict.get('hdu', defaultOption)[0], zojNum=dataDict.get('zoj', defaultOption)[0], cfNum=int(dataDict.get('codeforces', defaultOption)[0]) + int(dataDict.get('CodeForces', defaultOption)[0]), acdreamNum=dataDict.get('acdream', defaultOption)[0], bzojNum=dataDict.get('hysbz', defaultOption)[0] + dataDict.get('bzoj', defaultOption)[0], otherOJNum=tot - dataDict.get('poj', defaultOption)[0] - dataDict.get('hdu', defaultOption)[0] - dataDict.get('zoj', defaultOption)[0] - dataDict.get('codeforces', defaultOption)[0] - dataDict.get('acdream', defaultOption)[0] - dataDict.get('hysbz', defaultOption)[0] - dataDict.get('bzoj', defaultOption)[0], totalNum=tot, submitNum=submitTot, ratio=ratio, ) dataSource.append(saveData) from db import database # Insert rows 1000 at a time. with database.atomic(): for idx in range(0, len(dataSource), 1000): acRecordArchive.insert_many(dataSource[idx:idx + 1000]).execute() logging.log(20, '[CRON] Executed for %s Users', dataSource) print('Done')