def main():
node = models.Node.load(NODE_ID)
user = models.User.load(USER_ID)
emails = get_emails_since(TIME_DELTA)
sio = StringIO()
utils.make_csv(sio, emails, ['affiliation', 'count'])
utils.send_file(app, FILE_NAME, CONTENT_TYPE, sio, node, user)
def main():
node = models.Node.load(settings.TABULATE_EMAILS_NODE_ID)
user = models.User.load(settings.TABULATE_EMAILS_USER_ID)
emails = get_emails_since(settings.TABULATE_EMAILS_TIME_DELTA)
sio = StringIO()
utils.make_csv(sio, emails, ['affiliation', 'count'])
utils.send_file(settings.TABULATE_EMAILS_FILE_NAME, settings.TABULATE_EMAILS_CONTENT_TYPE, sio, node, user)
def main():
node = models.Node.load(settings.TABULATE_EMAILS_NODE_ID)
user = models.User.load(settings.TABULATE_EMAILS_USER_ID)
emails = get_emails_since(settings.TABULATE_EMAILS_TIME_DELTA)
sio = StringIO()
utils.make_csv(sio, emails, ['affiliation', 'count'])
utils.send_file(settings.TABULATE_EMAILS_FILE_NAME,
settings.TABULATE_EMAILS_CONTENT_TYPE, sio, node, user)
def main():
node = models.Node.load(NODE_ID)
user = models.User.load(USER_ID)
cutoff = datetime.datetime.utcnow() - TIME_OFFSET
result = run_map_reduce(query={'date': {'$gt': cutoff}})
sio = utils.make_csv(
((row['_id'], row['value'])
for row in result.find().sort([('value', pymongo.DESCENDING)])),
['name', 'count'],
)
utils.send_file(app, FILE_NAME, CONTENT_TYPE, sio, node, user)
def main():
node = models.Node.load(settings.TABULATE_LOGS_NODE_ID)
user = models.User.load(settings.TABULATE_LOGS_USER_ID)
cutoff = datetime.datetime.utcnow() - settings.TABULATE_LOGS_TIME_OFFSET
result = run_map_reduce(query={'date': {'$gt': cutoff}})
sio = StringIO()
utils.make_csv(
sio,
(
(row['_id'], row['value'])
for row in result.find().sort([('value', pymongo.DESCENDING)])
),
['name', 'count'],
)
utils.send_file(app, settings.TABULATE_LOGS_FILE_NAME, settings.TABULATE_LOGS_CONTENT_TYPE, sio, node, user)
def main():
node = models.Node.load(settings.TABULATE_LOGS_NODE_ID)
user = models.User.load(settings.TABULATE_LOGS_USER_ID)
cutoff = datetime.datetime.utcnow() - settings.TABULATE_LOGS_TIME_OFFSET
result = run_map_reduce(query={'date': {'$gt': cutoff}})
sio = StringIO()
utils.make_csv(
sio,
(
(row['_id'], row['value'])
for row in result.find().sort([('value', pymongo.DESCENDING)])
),
['name', 'count'],
)
utils.send_file(settings.TABULATE_LOGS_FILE_NAME, settings.TABULATE_LOGS_CONTENT_TYPE, sio, node, user)
def create_or_update_file(file_stream, file_name, self_id, parent_path,
parent_id):
path = '/'
if parent_path is not None:
path = '/' + parent_id + '/'
if self_id is None:
name = file_name
create = True
else:
name = self_id
create = False
path = '/'
node = models.Node.load(settings.TABULATE_LOGS_NODE_ID)
user = models.User.load(settings.TABULATE_LOGS_USER_ID)
retry = 0
while retry < 3: # server 503 may still happen with sleep(1), retry for 3 times
resp = utils.send_file(name, 'file-upload', file_stream, node, user,
create, path)
sleep(1) # walk around 503 error
if resp.status_code != 503:
break
retry += 1
if retry == 3:
logger.debug('response = {}'.format(resp.json()))
resp.raise_for_status()
def create_dir(dir_name, self_id, parent_path, parent_id):
assert self_id is None, 'Cannot create directories that already exist'
path = '/'
if parent_path is not None and len(parent_path) > 0:
path = '/' + parent_id + '/'
name = dir_name
create = True
node = models.Node.load(settings.TABULATE_LOGS_NODE_ID)
user = models.User.load(settings.TABULATE_LOGS_USER_ID)
retry = 0
while retry < 3: # server 503 may still happen with sleep(1), retry for 3 times
resp = utils.send_file(name, 'folder-upload', None, node, user, create,
path)
sleep(1) # walk around 503 error
if resp.status_code != 503:
break
retry += 1
if retry == 3:
resp.raise_for_status()