def main():
mail = imaplib.IMAP4_SSL(IMAP_SERVER)
mail.login(EMAIL_ACCOUNT, PASSWORD)
rv, data = mail.select(EMAIL_FOLDER)
if rv == 'OK':
logging.info("Processing mailbox: %s" % EMAIL_FOLDER)
process_mailbox(mail)
mail.close()
else:
logging.warining("ERROR: Unable to open mailbox %s " % rv)
mail.logout()
def crawlingData(self):
for code in self.code_list:
self.cur_code = code
try:
cur_url = self.batch_api + "code=" + code + "&start=" + self.begin_day + "&end=" + self.end_day
except Exception, e:
logging.warining(
"Please set the 'begin_day' and 'end_day' for data crawling."
)
dest_file = self.csv_path + '/history/' + code + ".csv"
self._getURL(cur_url, dest_file)
time.sleep(0.2)
def __call__(self, request):
kw = None
if self._has_var_kw_arg or self._has_named_kw_args or self._required_kw_args:
if request.method == 'POST':
if not request.content_type:
return web.HTTPBadRequest('Missing Content-Type.')
ct = request.content_type.lower()
if ct.startswith('application/json'):
params = yield from request.json()
if not isinstance(params, dict):
return web.HTTPBadRequest('JSON body must be object.')
kw = params
elif ct.startswith('application/x-www-form-urlencoded'
) or ct.startswith('multipart/form-data'):
params = yield from request.post()
kw = dict(**params)
else:
return web.HTTPBadRequest('Unsupported Contect-Type: %s' %
request.content_type)
if request.method == 'GET':
qs = request.query_string
if qs:
kw = dict()
for k, v in parse.parse_qs(qs, True).items():
kw[k] = v[0]
if kw is None:
kw = dict(**request.match_info)
else:
if not self._has_var_kw_arg and self._named_kw_args:
copy = dict()
for name in self._named_kw_args:
if name in kw:
copy[name] = kw[name]
kw = copy
for k, v in request.match_info.items():
if k in kw:
logging.warining(
'Duplicate arg name in named arg and kw args: %s' % k)
kw[k] = v
if self._has_request_arg:
kw['request'] = request
if self._required_kw_args:
for name in self._required_kw_args:
if not name in kw:
return web.HTTPBadRequest('Missing arguments: %s' % name)
logging.info('call with args: %s' % str(kw))
try:
r = yield from self._func(**kw)
return r
except APIError as e:
return dict(error=e.error, data=e.data, message=e.message)
def __call__(self, request):
kw = None
if self._has_var_kw_arg or self._has_named_kw_args or self._required_kw_args:
if request.method == 'POST':
if not request.content_type:
return web.HTTPBadRequest('Missing Content-Type.')
ct = request.content_type.lower()
if ct.startswith('application/json'):
params = yield from request.json()
if not isinstance(params, dict):
return web.HTTPBadRequest('JSON body must be object.')
kw = params
elif ct.startswith('application/x-www-form-urlencoded') or ct.startswith('multipart/form-data'):
params = yield from request.post()
kw = dict(**params)
else:
return web.HTTPBadRequest('Unsupported Contect-Type: %s' % request.content_type)
if request.method == 'GET':
qs = request.query_string
if qs:
kw = dict()
for k, v in parse.parse_qs(qs, True).items():
kw[k] = v[0]
if kw is None:
kw = dict(**request.match_info)
else:
if not self._has_var_kw_arg and self._named_kw_args:
copy = dict()
for name in self._named_kw_args:
if name in kw:
copy[name] = kw[name]
kw = copy
for k, v in request.match_info.items():
if k in kw:
logging.warining('Duplicate arg name in named arg and kw args: %s' % k)
kw[k] = v
if self._has_request_arg:
kw['request'] = request
if self._required_kw_args:
for name in self._required_kw_args:
if not name in kw:
return web.HTTPBadRequest('Missing arguments: %s' % name)
logging.info('call with args: %s' % str(kw))
try:
r = yield from self._func(**kw)
return r
except APIError as e:
return dict(error = e.error, data = e.data, message = e.message)
def set_candidates():
payload = request.json
if payload is None:
logging.warining("payload is invalid at request /set_candidates")
return resulty(False)
idx = payload.get("content_idx")
if idx is None:
logging.warning("payload is invalid at request /set_candidates")
return resulty(False)
if not payload.get("candidates"):
return resulty(False)
print("candidate found")
candidates = payload.get("candidates")
for candidate in candidates:
server.keyword_db.insert(candidate)
server.insert_candidates(idx, candidates)
return resulty(True)
def remove(self):
args = (self.getValueOrDefault(self.__primary_key__))
rows = yield from execute(self.__delete__,args)
if rows!=1:
logging.warining('failed to delete a row from table {tb_name}, affected rows: {rows}'.format(tb_name=self.__table__,rows=rows))
df['year'] = df['time'].apply(lambda x: x.year)
df['month'] = df['time'].apply(lambda x: x.month)
df['day'] = df['time'].apply(lambda x: x.day)
df['sentiment'] = -99
# write
if args.output:
output_name = args.output
else:
output_name = args.input.replace('.csv', '_cleaning.csv')
logging.info(f'Writing {output_name}')
df.to_csv(output_name)
# write training csv
if args.training:
training_name = args.input.replace('.csv', '_training.csv')
not_training_name = args.input.replace('.csv', '_not_training.csv')
frac_training = float(args.trainingEntries) / df.shape[0]
if frac_training > 1:
frac_training = 1
logging.warining(
'Requesting more training events than total number of events')
logging.info(f'Fraction of entries for training: {frac_training}')
mask = np.random.rand(len(df)) < frac_training
df_training = df[mask]
df_not_training = df[~mask]
logging.info(f'Writing {training_name}')
df_training.to_csv(training_name)
logging.info(f'Writing {not_training_name}')
df_not_training.to_csv(not_training_name)