async def post(self):
try:
file = self.request.files['file'][0]
except (KeyError, IndexError):
raise MissingArgumentError('file')
with tempfile.NamedTemporaryFile(
'wb',
prefix='taguette_import_',
suffix='.sqlite3',
) as tmp:
# Write the database to temporary file
tmp.write(file.body)
tmp.flush()
project_id = self.get_body_argument('project_id', None)
if project_id is None:
return await self._list_projects(tmp.name)
else:
try:
project_id = int(project_id)
except ValueError:
self.set_status(400)
return await self.send_json({
'error': "Invalid project ID",
})
return await self._import_project(tmp.name, project_id)
def get(self, *args, **kwargs):
name = self.get_argument('name', None)
affiliation = self.get_argument('affiliation', None)
if name is None and affiliation is None:
# Retrieve the list of all the StudyPerson
sp = [{
'name': p.name,
'affiliation': p.affiliation
} for p in StudyPerson.iter()]
self.write(json_encode(sp))
self.finish()
elif name is not None and affiliation is not None:
try:
p = StudyPerson.from_name_and_affiliation(name, affiliation)
except QiitaDBLookupError:
self.fail('Person not found', 404)
return
self.write({
'address': p.address,
'phone': p.phone,
'email': p.email,
'id': p.id
})
self.finish()
else:
arg_name = 'name' if name is None else 'affiliation'
raise MissingArgumentError(arg_name)
async def post(self):
def string_to_file(string):
# Helper function to convert script typed into field into a tempfile for job submission.
# Takes in the script contents of the field.
# Returns path to tempfile.
tempscript = tempfile.mkstemp()
file_object = open(tempscript[1], 'w')
file_object.write(string)
file_object.close()
return tempscript[1]
inputType = self.get_query_argument('inputType')
outputDir = self.get_query_argument('outputDir', default='')
sbatch_command = 'sbatch' + ' '
# Have two options to specify SLURM script in the request body: either with a path to the script, or with the script's text contents
if inputType:
if inputType == 'path':
script_path = self.get_body_argument('input')
try:
stdout, stderr, returncode = await self.run_command(
sbatch_command + script_path, cwd=outputDir)
errorMessage = ""
except Exception as e:
stdout, stderr, returncode, errorMessage = (
"",
"Something went wrong. Check console for more details.",
1, str(e))
elif inputType == 'contents':
script_contents = self.get_body_argument('input')
tempscript = string_to_file(script_contents)
try:
stdout, stderr, returncode = await self.run_command(
sbatch_command,
stdin=open(tempscript, 'rb'),
cwd=outputDir)
errorMessage = ""
except Exception as e:
stdout, stderr, returncode, errorMessage = (
"",
"Something went wrong. Check console for more details.",
1, str(e))
os.remove(tempscript)
else:
raise Exception(
'The query argument inputType needs to be either \'path\' or \'contents\'.'
)
else:
raise MissingArgumentError('inputType')
if stdout:
responseMessage = "Success: " + stdout
else:
responseMessage = "Failure: " + stderr
# jobID = re.compile('([0-9]+)$').search(stdout).group(1)
self.finish({
"responseMessage": responseMessage,
"returncode": returncode,
"errorMessage": errorMessage
})
def get_arg(self, value, name, clazz, default):
if value:
return clazz(value)
else:
if default != '__NONE__':
return default
raise MissingArgumentError(name)
def parse_json_arguments(self, *enforced_keys, **optional_keys):
"""Parse JSON argument like `get_argument`."""
if O_O.debug:
dump_in(f'Input: {self.request.method} {self.request.path}',
self.request.body.decode()[:500])
try:
req = json.loads(self.request.body.decode('utf-8'))
except json.JSONDecodeError as exception:
dump_error(self.request.body.decode())
raise ParseJSONError(exception.doc)
if not isinstance(req, dict):
dump_error(self.request.body.decode())
raise ParseJSONError('Req should be a dictonary.')
for key in enforced_keys:
if key not in req:
dump_error(self.request.body.decode())
raise MissingArgumentError(key)
req['remote_ip'] = self.request.remote_ip
req['request_time'] = int(time.time())
return Arguments(req)
def get_argument(self, name, default=_ARG_DEFAULT, strip=True):
args = self.get_arguments(name, strip=strip)
if not args:
if default is self._ARG_DEFAULT:
raise MissingArgumentError(name)
return default
return args[-1]
async def post(self):
inputType = self.get_query_argument('inputType')
outputDir = self.get_query_argument('outputDir', default='')
sbatch_command = self.sbatch + ' '
# Have two options to specify SLURM script in the request body: either with a path to the script, or with the script's text contents
if inputType:
if inputType == 'path':
if self.request.headers['Content-Type'] == 'application/json':
script_path = json.loads(self.request.body)["input"]
else:
script_path = self.get_body_argument('input')
try:
stdout, stderr, returncode = await self.run_command(
sbatch_command + script_path, cwd=outputDir)
errorMessage = ""
except Exception as e:
stdout, stderr, returncode, errorMessage = (
"",
"Something went wrong. Check console for more details.",
1, str(e))
elif inputType == 'contents':
if self.request.headers['Content-Type'] == 'application/json':
script_contents = json.loads(self.request.body)["input"]
else:
script_contents = self.get_body_argument('input')
with tempfile.TemporaryFile(mode='w+b',
dir=self.temp_dir) as temp:
temp.write(str.encode(script_contents))
temp.flush()
temp.seek(0)
try:
stdout, stderr, returncode = await self.run_command(
sbatch_command, stdin=temp.fileno(), cwd=outputDir)
errorMessage = ""
except Exception as e:
stdout, stderr, returncode, errorMessage = (
"",
"Something went wrong. Check console for more details.",
1, str(e))
else:
raise Exception(
'The query argument inputType needs to be either \'path\' or \'contents\'.'
)
else:
raise MissingArgumentError('inputType')
if stdout:
responseMessage = "Success: " + stdout
else:
responseMessage = "Failure: " + stderr
# jobID = re.compile('([0-9]+)$').search(stdout).group(1)
self.finish({
"responseMessage": responseMessage,
"returncode": returncode,
"errorMessage": errorMessage
})
def get_json_argument(self, name, default: Any = _ARG_DEFAULT):
if not hasattr(self.request, "json_argument"):
self.load_json()
if name not in self.request.json_argument:
if isinstance(default, _ArgDefaultMarker):
raise MissingArgumentError(name)
return default
arg = self.request.json_argument[name]
return arg
def get(self, base):
"""
:param base:
:return:
"""
try:
user_id = self.get_argument('user', default=None, strip=True)
id_estrangeiro = self.get_argument('cpf', default=None, strip=True)
product = self.get_argument('product', default=None, strip=True)
partner = self.get_argument('partner', default=None, strip=True)
key = self.get_argument('key', default=None, strip=True)
parceiro_id = self.get_argument(
'parceiro_id', default=None, strip=True)
tabela = 'VIEW_SSO_USER_DATA'
fields = (
'chave', 'cliente_id', 'nome', 'email', 'data_suspensao', 'data_cancelamento', 'produto_id',
'parceiro_id',
'id_estrangeiro')
sql = "select %s from %s "
db = self.db.get(base)
if key is not None:
sql = sql % (','.join(fields), tabela) + ' where chave=%s'
qpar = [key]
elif partner is not None:
sql = sql % (','.join(fields),
tabela) + ' where parceiro_id=%s and cliente_id=%s and data_cancelamento is Null and data_suspensao is Null'
qpar = [partner, user_id]
elif id_estrangeiro is not None and parceiro_id is not None:
sql = sql % (','.join(fields),
tabela) + ' where parceiro_id=%s and id_estrangeiro=%s and data_cancelamento is Null and data_suspensao is Null'
qpar = [parceiro_id, id_estrangeiro]
elif id_estrangeiro is not None:
sql = sql % (','.join(fields),
tabela) + ' where produto_id=%s and id_estrangeiro=%s and data_cancelamento is Null and data_suspensao is Null'
qpar = [product, id_estrangeiro]
elif product is not None and user_id is not None:
sql = sql % (','.join(fields),
tabela) + ' where produto_id=%s and cliente_id=%s and data_cancelamento is Null and data_suspensao is Null'
qpar = [product, user_id]
else:
raise MissingArgumentError('data')
cur = yield db.fexecute(sql, qpar, table=tabela, cnames=fields, expires=60)
if cur and cur.value in ['', False, 0, None]:
raise DoesNotExist
self.success(json_formats(dict(zip(fields, cur.value))), True)
except (DoesNotExist, TypeError):
self.fail('Client Id not found', code=404)
except MissingArgumentError as e:
self.fail(e.log_message, code=e.status_code)
except Exception as e:
slog.error(e)
self.error('General Oauth Error', code=500)
def process(ctlr):
token_params = Arguments(ctlr.get_token())
now = int(time.time())
if not token_params:
raise MissingArgumentError('unvalid token.')
if 'timestamp' not in token_params:
raise MissingArgumentError('no timestamp info in token.')
if token_params.timestamp < now:
raise TokenExpiredError()
params = dict()
params['token'] = token_params
params['device'] = ctlr.get_argument('device', 'web')
params['lang'] = ctlr.get_argument('lang', 'cn').lower()
params['remote_ip'] = ctlr.request.remote_ip
params['request_time'] = now
ctlr.params = Arguments(params)
def get_json_argument(self,
name: str,
default=RequestHandler._ARG_DEFAULT):
try:
return self.json_arguments[name]
except KeyError:
if default is RequestHandler._ARG_DEFAULT:
raise MissingArgumentError(name)
else:
return default
async def post(self, project_id):
project, privileges = self.get_project(project_id)
if not privileges.can_add_document():
return await self.send_error_json(403, "Unauthorized")
try:
name = self.get_body_argument('name')
validate.document_name(name)
description = self.get_body_argument('description')
validate.description(description)
try:
file = self.request.files['file'][0]
except (KeyError, IndexError):
raise MissingArgumentError('file')
content_type = file.content_type
filename = validate.filename(file.filename)
direction = self.get_body_argument('text_direction',
'LEFT_TO_RIGHT')
try:
direction = database.TextDirection[direction]
except KeyError:
return await self.send_error_json(400,
"Invalid text direction")
try:
body = await convert.to_html_chunks(
file.body, content_type, filename,
self.application.config,
)
except convert.ConversionError as err:
return await self.send_error_json(400, str(err))
else:
doc = database.Document(
name=name,
description=description,
filename=filename,
project=project,
text_direction=direction,
contents=body,
)
self.db.add(doc)
self.db.flush() # Need to flush to get doc.id
cmd = database.Command.document_add(
self.current_user,
doc,
)
self.db.add(cmd)
logger.info("Document added to project %r: %r %r (%d bytes)",
project.id, doc.id, doc.name, len(doc.contents))
self.db.commit()
self.db.refresh(cmd)
self.application.notify_project(project.id, cmd)
return await self.send_json({'created': doc.id})
except validate.InvalidFormat as e:
logger.info("Error validating DocumentAdd: %r", e)
return await self.send_error_json(400, self.gettext(e.message))
def get_argument(self,
name: str,
default=RequestHandler._ARG_DEFAULT,
strip: bool = True,
cast: type = None):
try:
value = super().get_argument(name, strip=True)
return cast(value) if cast else value
except MissingArgumentError:
if default is RequestHandler._ARG_DEFAULT:
raise MissingArgumentError(name)
else:
return default
def get_json_argument(self, name, default=None):
try:
args = tornado.escape.json_decode(self.request.body)
name = tornado.escape.to_unicode(name)
if name in args:
return args[name]
elif default is not None:
return default
else:
raise MissingArgumentError(name)
except:
return self.get_argument(name, '')
async def post(self, project_id):
PROM_API.labels('document_add').inc()
project, privileges = self.get_project(project_id)
if not privileges.can_add_document():
self.set_status(403)
return self.send_json({'error': "Unauthorized"})
try:
name = self.get_body_argument('name')
validate.document_name(name)
description = self.get_body_argument('description')
validate.document_description(description)
try:
file = self.request.files['file'][0]
except (KeyError, IndexError):
raise MissingArgumentError('file')
content_type = file.content_type
filename = validate.filename(file.filename)
try:
body = await convert.to_html_chunks(file.body, content_type,
filename)
except convert.ConversionError as err:
self.set_status(400)
return self.send_json({
'error': str(err),
})
else:
doc = database.Document(
name=name,
description=description,
filename=filename,
project=project,
contents=body,
)
self.db.add(doc)
self.db.flush() # Need to flush to get doc.id
cmd = database.Command.document_add(
self.current_user,
doc,
)
self.db.add(cmd)
logger.info("Document added to project %r: %r %r (%d bytes)",
project.id, doc.id, doc.name, len(doc.contents))
self.db.commit()
self.db.refresh(cmd)
self.application.notify_project(project.id, cmd)
return self.send_json({'created': doc.id})
except validate.InvalidFormat as e:
logging.info("Error validating DocumentAdd: %r", e)
self.set_status(e.status_code, e.reason)
return self.send_json({'error': e.message})
def open(self):
try:
self.name = self.get_argument(name='name')
if not self.name.strip():
raise MissingArgumentError(arg_name='name')
except MissingArgumentError:
send_message(self, error='Please tell us your name.')
else:
game_id = self.get_argument(name='game', default=None)
if game_id is None:
# if enough games are active this can loop infinitely
# but 36^4 = 1.6million which means that is not a
# realistic scenario in any near-term timeframe
while game_id is None or game_id in _game_map:
game_id = u''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(4))
_game_map[game_id] = set()
self.is_host = True
else:
# treat all user-typed game_ids as case insensitive
game_id = game_id.upper()
try:
if len(self.name) > MAX_NAME_LENGTH:
raise ValueError('Please restart with a shorter name.')
current_names = {player.name for player in _game_map[game_id]}
if self.name in current_names:
raise ValueError('That name is already taken.')
arbitrary_player = next(iter(_game_map[game_id]), None)
if arbitrary_player and arbitrary_player.game_state is not None:
raise ValueError('This game is already in progress.')
_game_map[game_id].add(self)
self.game_id = game_id
except KeyError:
send_message(self, error='Game {} not found.'.format(game_id))
except ValueError as e:
send_message(self, error=str(e))
else:
broadcast_message(
self,
message='login',
data={
'game':
self.game_id,
'players':
[player.name for player in _game_map[self.game_id]]
})
def get_current_user(self):
try:
return self.get_secure_cookie("user_id").decode('utf-8')
except AttributeError:
try:
token_id = self.request.headers.get('token-id')
token_secret = self.request.headers.get('token-secret')
with self.control.session as session:
token = session.query(Token)\
.filter(Token.id == token_id)\
.one()
token.authenticate(token_secret)
session.commit()
return token.user_id
except ValueError:
raise MissingArgumentError('Not already logged in or incorrect\
auth id and token provided.')
def get(self):
file_path = self.get_query_argument("file", None)
if file_path is None:
raise MissingArgumentError("File argument is required")
path = self.get_query_argument("path", None)
format_arg = self.get_query_argument("format", None)
with h5py.File(as_absolute_path(self.base_dir, Path(file_path)),
"r") as h5file:
content = self.get_content(h5file, path)
response = encode(content, format_arg)
for key, value in response.headers.items():
self.set_header(key, value)
self.finish(response.content)
def get_arg(self, name, default=_arg_default, first=False):
'''
Returns the value of the argument with the given name. Similar to
``.get_argument`` but uses ``self.args`` instead.
If default is not provided, the argument is considered to be
required, and we raise a `MissingArgumentError` if it is missing.
If the argument is repeated, we return the last value. If ``first=True``
is passed, we return the first value.
``self.args`` is always UTF-8 decoded unicode. Whitespaces are stripped.
'''
if name not in self.args:
if default is _arg_default:
raise MissingArgumentError(name)
return default
return self.args[name][0 if first else -1]
async def post(self):
def string_to_file(string):
file_object = open('temporary_file.temporary', 'w')
file_object.write(string)
file_object.close()
return
self.request_log_function()
scriptIs = self.get_query_argument('scriptIs')
# Have two options to specify SLURM script in the request body: either with a path to the script, or with the script's text contents
if scriptIs:
if scriptIs == 'path':
script_path = self.get_body_argument('script')
stdout, stderr = await self.run_command('sbatch ' +
script_path)
elif scriptIs == 'contents':
self.log.debug('Body arguments: ' +
str(self.request.body_arguments))
script_contents = self.get_body_argument('script')
self.log.debug('script_contents: ' + script_contents)
string_to_file(script_contents)
stdout, stderr = await self.run_command(
'sbatch', stdin=open('temporary_file.temporary', 'rb'))
import os
os.remove('temporary_file.temporary')
else:
self.log.debug('Body arguments: ' +
str(self.request.body_arguments))
self.log.debug('Query arguments: ' +
str(self.request.query_arguments))
raise Exception(
'The query argument scriptIs needs to be either \'path\' or \'contents\'.'
)
else:
self.log.debug('Body arguments: ' +
str(self.request.body_arguments))
self.log.debug('Query arguments: ' +
str(self.request.query_arguments))
raise MissingArgumentError('scriptIs')
jobID = re.compile('([0-9]+)$').search(stdout).group(1)
self.finish(jobID)
def wrapper(self, *args, **kwargs):
if name_old is not None:
file_old = self.get_argument(name_old, None)
else:
file_old = None
try:
argument = self.request.files.get(name)
if not argument and not file_old:
raise MissingArgumentError(name)
except HTTPError as error:
if isinstance(error, MissingArgumentError) and not required:
argument = _missing
else:
if status_code != 400 or response_body != None: # 定制返回结果
kanjian_httperror = KanjianHTTPError.from_httperror(
error,
status_code=status_code,
response_body=response_body)
raise kanjian_httperror
else:
raise error
return get_or_post(self, *args, **kwargs)
def parse_json_arguments(self, key_list):
"""Parse JSON argument like `get_argument`."""
try:
if config.debug:
sys.stdout.write('\n\n' + '>' * 80)
sys.stdout.write('\n' + (f'Input: '
f'{self.request.method} '
f'{self.request.path}') + '\n\n')
sys.stdout.write(self.request.body.decode()[:500])
sys.stdout.write('\n\n' + '>' * 80 + '\n')
sys.stdout.flush()
req = json.loads(self.request.body.decode('utf-8'))
except json.JSONDecodeError as exception:
# self.dump_fail_data(
# exc_doc=exception.doc, msg=exception.args[0], status=1)
sys.stdout.write(self.request.body.decode())
sys.stdout.write('\n')
sys.stdout.flush()
raise ParseJSONError(exception.doc)
if not isinstance(req, dict):
sys.stdout.write(self.request.body.decode())
sys.stdout.write('\n')
sys.stdout.flush()
raise ParseJSONError('Req should be a dictonary.')
for key in list(req.keys()):
req[camel_to_underline(key)] = req[key]
for key in key_list:
if key not in req:
sys.stdout.write(self.request.body.decode())
sys.stdout.write('\n')
sys.stdout.flush()
raise MissingArgumentError(key)
req['user_ip'] = self.request.remote_ip
return Arguments(req)
def parse_json_arguments(self, **keys):
"""Parse JSON argument like `get_argument`."""
try:
if config.debug:
sys.stdout.write('\n\n' + '>' * 80)
sys.stdout.write('\n' + (f'Input: '
f'{self.request.method} '
f'{self.request.path}') + '\n\n')
sys.stdout.write(self.request.body.decode()[:500])
sys.stdout.write('\n\n' + '>' * 80 + '\n')
sys.stdout.flush()
req = json.loads(self.request.body.decode('utf-8'))
except json.JSONDecodeError as exception:
# self.fail(
# exc_doc=exception.doc, msg=exception.args[0], status=1)
sys.stdout.write(self.request.body.decode())
sys.stdout.write('\n')
sys.stdout.flush()
raise ParseJSONError(exception.doc)
if not isinstance(req, dict):
sys.stdout.write(self.request.body.decode())
sys.stdout.write('\n')
sys.stdout.flush()
raise ParseJSONError('Req should be a dictonary.')
for key in keys:
if keys[key] is ENFORCED and key not in req:
sys.stdout.write(self.request.body.decode())
sys.stdout.write('\n')
sys.stdout.flush()
raise MissingArgumentError(key)
req['remote_ip'] = self.request.remote_ip
req['request_time'] = int(time.time())
return Arguments(req)
def parse_args(self, arg_list):
results = {}
for arg in arg_list:
location = arg.get('location')
required = arg.get('required')
name = arg.get('name')
cast = arg.get('cast')
para = self.location(location)
if name not in para:
if 'default' in arg:
para[name] = arg['default']
elif required:
raise MissingArgumentError(name)
else:
continue
value = para.get(name)
if cast:
try:
value = cast(value)
except Exception, e:
raise HTTPError(400, "Invalid %s(%s): %s" % (str(cast), name, e))
results[name] = value
async def post(self):
def string_to_file(string):
file_object = open('temporary_file.temporary', 'w')
file_object.write(string)
file_object.close()
return
inputType = self.get_query_argument('inputType')
# Have two options to specify SLURM script in the request body: either with a path to the script, or with the script's text contents
if inputType:
if inputType == 'path':
script_path = self.get_body_argument('input')
stdout, stderr, returncode = await self.run_command(
'sbatch ' + script_path)
elif inputType == 'contents':
script_contents = self.get_body_argument('input')
string_to_file(script_contents)
stdout, stderr, returncode = await self.run_command(
'sbatch', stdin=open('temporary_file.temporary', 'rb'))
os.remove('temporary_file.temporary')
else:
raise Exception(
'The query argument inputType needs to be either \'path\' or \'contents\'.'
)
else:
raise MissingArgumentError('inputType')
if stdout:
responseMessage = stdout
else:
responseMessage = stderr
# jobID = re.compile('([0-9]+)$').search(stdout).group(1)
self.finish({
"responseMessage": responseMessage,
"returncode": returncode
})
def my_get_argument(name, val=NOMEAN):
if body.get(name, val) == NOMEAN:
raise MissingArgumentError(name)
return body.get(name, val)
def post(self, *args, **keywords):
task_type = args[0]
try:
from drenaj_api.utils.drenajneo4jmanager import init_user_to_graph_aux, upsert_campaign
task_definition = bson.json_util.loads(
self.get_argument('task_definition'))
queue = self.get_argument('queue')
metadata = task_definition['metadata']
campaign_id = self.get_argument('campaign_id', 'default')
campaign_node = upsert_campaign(campaign_id)
user = init_user_to_graph_aux(campaign_node, metadata['user'])
# user = self.application.db.get_user_from_watchlist(metadata['user'])
print user
if task_type == 'timeline':
timeline_task_state = [
rel for rel in user.match_incoming('TIMELINE_TASK_STATE')
][0]
user_object = dict(user.properties)
user_object.update({'campaign_ids': campaign_id})
res = app_object.send_task('timeline_retrieve_userlist', [[[
user_object,
timeline_task_state.properties['since_tweet_id'],
timeline_task_state.properties['page_not_found']
]]],
queue=queue)
timeline_task_state.properties['state'] = 1
timeline_task_state.push()
elif task_type == 'friendfollower':
friendfollower_task_state = \
[rel for rel in user.match_incoming('FRIENDFOLLOWER_TASK_STATE')][0]
res = app_object.send_task('crawl_friends_or_followers', [[[
dict(user.properties),
friendfollower_task_state.properties['page_not_found']
]]],
queue=queue)
friendfollower_task_state.properties['state'] = 1
friendfollower_task_state.push()
elif task_type == 'userinfo':
userinfo_task_state = \
[rel for rel in user.match_incoming('USER_INFO_HARVESTER_TASK_STATE')][0]
res = app_object.send_task('crawl_user_info',
[[[dict(user.properties), 0]]],
queue=queue)
userinfo_task_state.properties['state'] = 1
userinfo_task_state.push()
else:
raise MissingArgumentError('timeline or friendfollower')
self.write(
bson.json_util.dumps({
'result': 'successful',
'message': res.task_id
}))
self.add_header('Content-Type', 'application/json')
except MissingArgumentError as e:
raise HTTPError(
500, 'You didn'
't supply %s as an argument' % e.arg_name)
def get_argument(self, name):
if name in self.arguments:
return self.arguments[name]
raise MissingArgumentError(name)
def _get_argument_or_raise(self, data, key):
try:
return data[key]
except KeyError:
raise MissingArgumentError(key)
def post(self, *args, **kwargs):
data = json.loads(self.request.body.decode('utf-8'))
_ids = data.get('_ids', None)
force_push = data.get('force_push', False)
if not _ids:
raise MissingArgumentError("_ids")
coll = self.application.db[self.collection]
if force_push:
audios = yield [
coll.find_one({"_id": ObjectId(_id)}) for _id in _ids
]
else:
audios = yield [
coll.find_one({
"_id": ObjectId(_id),
"sendToCNRTime": None
}) for _id in _ids
]
# 如果对应的audio与媒体文件没有被下载，那么下载对应的audio与媒体文件
audiosInfo = None
imgsInfo = None
if self.web_str == 'kl':
# 因为kl网站挂掉了所以暂时不提供kl网站下载
raise UnSupportWebError(self.web_str)
elif self.web_str == 'xmly':
# 因为取消掉媒体文件下载进程，所以所有媒体文件下载都在这里
audios_url = [audio.get("play_path", None) for audio in audios]
imgs_url = [audio.get("cover_url_142", None) for audio in audios]
audiosInfo = yield [
self.xmlyAudioDownloader.download_file(url)
for url in audios_url
]
imgsInfo = yield [
self.xmlyImgDownloader.download_file(url) for url in imgs_url
]
elif self.web_str == 'qt':
audios_url = [audio.get("playUrl") for audio in audios]
audiosInfo = yield [
self.qtAudioDownloader.download_file(url) for url in audios_url
]
# 因为爬虫没有获得img url，所以imgs 都为空
imgsInfo = [None for audio in audios]
else:
raise UnSupportWebError(self.web_str)
audiosInfo = zip(audios, audiosInfo, imgsInfo)
xmls = [
xmlGenerator.getXMLContentFromAudio(self.web_str, audioInfo)
for audioInfo in audiosInfo if audiosInfo[0]
]
resps = yield [self.sendXMLToCNR(xml) for xml in xmls]
# 将推送到cnr 的时间设置到数据库中
yield [
coll.update({"_id": audio["_id"]},
{"$set": {
"sendToCNRTime": datetime.datetime.now()
}}) for audio in audios
]
self.write({
"audios": [audio.get('album_title') for audio in audios],
"resps": ['success' if resp else 'fault' for resp in resps],
"request_push_count": len(_ids),
"real_push_count": len(xmls),
"force_push": force_push,
})
