def parser_artist(artist_id):
create_app()
process = Process.get_or_create(id=artist_id) # Process以歌手为单位
if process.is_success:
return
print('Starting fetch artist: {}'.format(artist_id))
start = time.time()
process = Process.get_or_create(id=artist_id)
tree = get_tree(ARTIST_URL.format(artist_id)) # 使用requests获取页面文本，转化为lxml对象
artist = Artist.objects.filter(id=artist_id)
if not artist: # 如果之前没抓过
artist_name = tree.xpath('//h2[@id="artist-name"]/text()')[0]
picture = tree.xpath(
'//div[contains(@class, "n-artist")]//img/@src')[0]
artist = Artist(id=artist_id, name=artist_name, picture=picture)
artist.save()
else: # 如果之前抓过，但是该歌手的歌曲没抓完
artist = artist[0]
song_items = tree.xpath('//div[@id="artist-top50"]//ul/li/a/@href')
songs = []
for item in song_items:
song_id = item.split('=')[1]
song = parser_song(song_id, artist) # 进入抓取和解析歌手模式
if song is not None:
songs.append(song)
artist.songs = songs
artist.save()
process.make_succeed() # 标记歌手下的热门歌曲的热门评论抓完
print('Finished fetch artist: {} Cost: {}'.format(artist_id,
time.time() - start))
def parser_artist(artist_id):
create_app()
process = Process.get_or_create(id=artist_id)
if process.is_success:
return
print 'Starting fetch artist: {}'.format(artist_id)
start = time.time()
process = Process.get_or_create(id=artist_id)
tree = get_tree(ARTIST_URL.format(artist_id))
artist = Artist.objects.filter(id=artist_id)
if not artist:
artist_name = tree.xpath('//h2[@id="artist-name"]/text()')[0]
picture = tree.xpath(
'//div[contains(@class, "n-artist")]//img/@src')[0]
artist = Artist(id=artist_id, name=artist_name, picture=picture)
artist.save()
else:
artist = artist[0]
song_items = tree.xpath('//div[@id="artist-top50"]//ul/li/a/@href')
songs = []
for item in song_items:
song_id = item.split('=')[1]
song = parser_song(song_id, artist)
if song is not None:
songs.append(song)
artist.songs = songs
artist.save()
process.make_succeed()
print 'Finished fetch artist: {} Cost: {}'.format(
artist_id, time.time() - start)
def get_movie_info(movie_id):
create_app()
process = Process.get_or_create(id=movie_id)
if process.is_success:
return
print 'Strting fetch movie: {}'.format(movie_id)
start = time.time()
process = Process.get_or_create(id=movie_id)
movie = Movie.objects.filter(id=movie_id)
if not movie:
html = get_tree(MOVIE_URL.format(movie_id))
name = html.xpath("//div[@id='content']//h1/span/text()")[0]
mark = html.xpath(
"//div[@class='rating_wrap clearbox']//strong/text()")[0]
picture = html.xpath(
"//div[@id='content']//div[@id='mainpic']//img/@src")[0]
movie = Movie(id=movie_id, name=name, mark=mark, picture=picture)
movie.save()
get_top_comment_and_user_info(movie_id, movie)
process.make_succeed()
print 'Finished fetch movie: {} Cost: {}'.format(
movie_id,
time.time() - start)
def parser_artist(artist_id):
create_app()
process = Process.get_or_create(id=artist_id)
if process.is_success:
return
print 'Starting fetch artist: {}'.format(artist_id)
start = time.time()
process = Process.get_or_create(id=artist_id)
tree = get_tree(ARTIST_URL.format(artist_id))
artist = Artist.objects.filter(id=artist_id)
if not artist:
artist_name = tree.xpath('//h2[@id="artist-name"]/text()')[0]
picture = tree.xpath(
'//div[contains(@class, "n-artist")]//img/@src')[0]
artist = Artist(id=artist_id, name=artist_name, picture=picture)
artist.save()
else:
artist = artist[0]
song_items = tree.xpath('//div[@id="artist-top50"]//ul/li/a/@href')
songs = []
for item in song_items:
song_id = item.split('=')[1]
song = parser_song(song_id, artist)
if song is not None:
songs.append(song)
artist.songs = songs
artist.save()
process.make_succeed()
print 'Finished fetch artist: {} Cost: {}'.format(artist_id,
time.time() - start)
def parser_artist(artist_id):
create_app()
process = Process.get_or_create(id=artist_id)
if process.is_success:
print "find process artist finished ,return"
return
print 'Starting fetch artist: {}'.format(artist_id)
start = time.time()
process = Process.get_or_create(id=artist_id)
tree = get_tree(ARTIST_URL.format(artist_id)) #get artist url
if tree==None:
print "fetch artist url get none,return !"
return
artist = Artist.objects.filter(id=artist_id)
if not artist:
print "create artist "+str(artist_id)
artist_name = tree.xpath('//h2[@id="artist-name"]/text()')[0]
picture = tree.xpath(
'//div[contains(@class, "n-artist")]//img/@src')[0]
artist = Artist(id=artist_id, name=artist_name, picture=picture)
artist.save()
else:
artist = artist[0]
print "artist exist " + str(artist_id)
print "fetching all song comments"
song_items = tree.xpath('//div[@id="artist-top50"]//ul/li/a/@href')
#song_items2=tree.xpath('//ul[@class="f-hide"]/li/a/@href') the same
songs = []
print song_items
if song_items==[]:
print "Artist get no songs ,return fetch artist {}".format(artist_id)
return
for item in song_items:
song_id = item.split('=')[1]
song = parser_song(song_id, artist)
if song is None:
print "parse song failed,return "
return
else:
songs.append(song)
artist.songs = songs
artist.save()
process.make_succeed()
print 'Finished fetch artist: {} Cost: {}'.format(
artist_id, time.time() - start)
def log_cur_window(self):
cur_class, cur_window, cur_name = self.get_cur_window()
if cur_class is None:
return
self.session = self.session_maker()
if cur_class != self.cur_class:
self.cur_class = cur_class
proc_name = self.cur_class.decode('latin1')
cur_process = self.session.query(Process).filter_by(
name=proc_name).scalar()
if cur_process is None:
cur_process = Process(proc_name)
self.session.add(cur_process)
self.trycommit()
self.cur_process_id = cur_process.id
if cur_window != self.cur_window or cur_name != self.cur_name:
self.cur_window = cur_window
self.cur_name = cur_name
self.store_keys()
self.store_window()
self.check_geometry()
def create(self, request, *args, **kwargs):
lti_session = getattr(self.request, 'LTI', {})
audit_user_id = lti_session.get('custom_canvas_user_login_id')
account_sis_id = lti_session.get('custom_canvas_account_sis_id')
if not all((audit_user_id, account_sis_id)):
raise DRFValidationError(
'Invalid LTI session: custom_canvas_user_login_id and '
'custom_canvas_account_sis_id required')
account = self.request.data.get('account')
term = self.request.data.get('term')
if not all((account, term)):
raise DRFValidationError('Both account and term are required')
# for the moment, only the current school account can be operated on
if not account_sis_id[len('school:'):] == account:
raise PermissionDenied
selected_courses = self.request.data.get('course_list')
process = Process.enqueue(
bulk_publish_canvas_sites,
settings.RQWORKER_QUEUE_NAME,
account='sis_account_id:school:{}'.format(account),
term='sis_term_id:{}'.format(term),
audit_user=audit_user_id,
course_list=selected_courses)
logger.debug('Enqueued Process job for bulk_publish_canvas_sites: '
'{}'.format(process))
return Response(self.serializer_class(process).data,
status=status.HTTP_201_CREATED)
def add_process(process_component: ProcessCreateModel,
db: Session = Depends(deps.get_db)):
"""
Create Process
"""
return jsonable_encoder(
Process.create(db=db, process_component=process_component))
def got_screen_change(self, process_name, window_name, win_x, win_y, win_width, win_height):
""" Receives a screen change and stores any changes. If the process or window has
changed it will also store any queued pressed keys.
process_name is the name of the process running the current window
window_name is the name of the window
win_x is the x position of the window
win_y is the y position of the window
win_width is the width of the window
win_height is the height of the window """
cur_process = self.session.query(Process).filter_by(name=process_name).scalar()
if not cur_process:
cur_process = Process(process_name)
self.session.add(cur_process)
cur_geometry = self.session.query(Geometry).filter_by(xpos=win_x,
ypos=win_y,
width=win_width,
height=win_height).scalar()
if not cur_geometry:
cur_geometry = Geometry(win_x, win_y, win_width, win_height)
self.session.add(cur_geometry)
cur_window = self.session.query(Window).filter_by(title=window_name,
process_id=cur_process.id).scalar()
if not cur_window:
cur_window = Window(window_name, cur_process.id)
self.session.add(cur_window)
if not (self.current_window.proc_id == cur_process.id
and self.current_window.win_id == cur_window.id):
self.trycommit()
self.store_keys() # happens before as these keypresses belong to the previous window
self.current_window.proc_id = cur_process.id
self.current_window.win_id = cur_window.id
self.current_window.geo_id = cur_geometry.id
def main(db):
result_qs = Process.get_queryset(
db=db).filter((Process.status == 'queued')
| (Process.status == 'in_progress'))
result_with_order_by = result_qs.order_by(Process.priority)
result = list(result_with_order_by)
print(f"Total {len(result)} Tasks Received.\n")
start = time.time()
if result:
result_qs.update({"status": "in_progress"})
db.commit()
tasks = []
loop = asyncio.get_event_loop()
for instance in result:
if instance.func_name == 'sum':
tasks.append(loop.create_task(add(instance, db)))
elif instance.func_name == 'diff':
tasks.append(loop.create_task(diff(instance, db)))
elif instance.func_name == 'multiply':
tasks.append(loop.create_task(multiply(instance, db)))
else:
instance.output = "Func not found"
instance.status = "failed"
if tasks:
loop.run_until_complete(asyncio.wait(tasks))
# loop.close()
db.commit()
end = time.time()
print(f'Completed Time: {end - start:.2f} sec\n\n')
def add_process(self, type, status, file_id=None,
transcription_id=None, api_version='v1',
asr_model=None):
process = Process(
file_id=file_id,
transcription_id=transcription_id,
type = type,
status = status,
api_version = api_version)
process.asr_model=asr_model
self.db.session.add(process)
self.db.session.flush()
self.db.session.commit()
return process
def index(request):
course_instance_id = request.LTI.get('lis_course_offering_sourcedid')
canvas_course_id = request.LTI.get('custom_canvas_course_id')
audit_user_id = request.LTI.get('custom_canvas_user_login_id')
if request.method == 'POST':
keyword = request.POST.get('keyword')
title = request.POST.get('title')
term = request.POST.get('term')
Process.enqueue(
migrate_files,
settings.ISITES_MIGRATION_QUEUE_NAME,
keyword=keyword,
canvas_course_id=canvas_course_id,
term=term,
title=title,
audit_user=audit_user_id,
)
# try to get the school.
# if we have a course instance id, try that first.
school = get_school(course_instance_id, canvas_course_id)
logger.info(u'"migration started" %s %s "%s" "%s" %s %s %s' % (request.user.username, keyword, title, term,
course_instance_id, canvas_course_id, school))
return redirect('isites_migration:index')
processes = Process.objects.filter(
name='isites_migration.jobs.migrate_files',
details__canvas_course_id=canvas_course_id
).order_by('-date_created')
has_active_process = len([p for p in processes if p.state != Process.COMPLETE]) > 0
return render(request, 'isites_migration/index.html', {
'isites': get_previous_isites(course_instance_id),
'processes': processes,
'link_to_files_page': settings.CANVAS_URL + '/courses/%s/files' % canvas_course_id,
'has_active_process': has_active_process,
# Harcode to EST for now, but eventually set based on the launch user's profile setting
'user_timezone': 'America/New_York',
})
def get(self, process_number):
if self._process_number_is_valid(process_number) is False:
return {
'msg': f'Número do processo {process_number} inválido'
}, 422
process = Process.query.get(process_number)
if process is None:
process = Process(process_number=process_number)
db.session.add(process)
db.session.commit()
crawler_task.delay(process_number, 1)
crawler_task.delay(process_number, 2)
process_schema = ProcessSchema()
return process_schema.jsonify(process)
def validate_processes(self, field):
if not self.processes.data:
raise ValidationError(u'Необходимо выбрать хотя бы один процесс')
# Дополнительная проверка выбора пользователя:
# Пользователю CSO необходимо выбрать хотя бы по одному процессу для каждого пользователя
# (это позволяет предотвратить возможные ошибки при входе пользователей)
process_owner_ids = []
for choice in self.processes.data:
owner = Process.process_by_id(choice).role_id
if not process_owner_ids.__contains__(owner):
process_owner_ids.append(owner)
# Идентификаторы тестируемых ролей начинаются с 3 и заканчиваются на 6
if not sorted(process_owner_ids) == [3, 4, 5, 6]:
raise ValidationError(u'Необходимо выбрать хотя бы по одному процессу '
u'для каждого тестируемого пользователя')
def post(self, request, format=None):
date = request.DATA.get('date')
try:
activity = Activity.objects.get(name=request.DATA.get('activity'))
except Activity.DoesNotExist:
activity = Activity(name=request.DATA.get('activity'), average_price=0)
activity.save()
p = Process(date=request.DATA.get('date'), activity=activity, average_price=0)
p.save()
consumer_list = json.loads(request.DATA.get('list'))
user_to_be_saved = []
total = 0
try:
for num, i in enumerate(consumer_list):
name = i.get('name')
prepay = i.get('prepay')
consume = i.get('consume')
#check name
user = User.objects.get(name=name)
p.participants.add(user)
#user money
user.remain += prepay - consume
user_to_be_saved.append(user)
total += consume
#send email
except:
print sys.exc_info()
p.delete()
return Response(status=404)
#caculate the average price of process
p.average_price =total/(num +1)
#caculate the average price of activity
if p.average_price == 0:
activity.average_price = p.average_price
else:
activity.average_price = (p.average_price + activity.average_price)/2
p.save()
for i in user_to_be_saved:
i.save()
return Response({"detail":"success"} ,status=200)
def record_status(processname, message=None, percent_done=None):
try:
proc_rec = Process.objects.get(name=processname)
except Process.DoesNotExist:
proc_rec = Process(name=processname)
if message:
proc_rec.statustext = message
if percent_done:
proc_rec.percentdone = percent_done
proc_rec.save()
def record_status(processname, message=None, percent_done=None):
'''Record process feedback so we can display it during long-running
operations'''
try:
proc_rec = Process.objects.get(name=processname)
except Process.DoesNotExist:
proc_rec = Process(name=processname)
if message:
proc_rec.statustext = message
if percent_done:
proc_rec.percentdone = percent_done
proc_rec.save()
def setUp(self):
super(BulkPublishCanvasSitesBaseTestCase, self).setUp()
self.account = 'colgsas'
self.term = '2017-1'
self.test_user_id = '12345678'
self.process = Process.enqueue(
bulk_publish_canvas_sites,
QUEUE_NAME,
account='sis_account_id:school:{}'.format(self.account),
term='sis_term_id:{}'.format(self.term),
audit_user=self.test_user_id,
course_list=None)
self._refresh_from_db()
self.assertIsNotNone(self.process.date_created)
self.assertIsNone(self.process.date_active)
self.assertEqual(self.process.state, Process.QUEUED)
self.assertEqual(self.process.status, '')
def test_publish_selected(self, *args):
self.process = Process.enqueue(
bulk_publish_canvas_sites,
QUEUE_NAME,
account='sis_account_id:school:{}'.format(self.account),
term='sis_term_id:{}'.format(self.term),
audit_user=self.test_user_id,
course_list=[123, 124, 125])
self._refresh_from_db()
self.assertIsNotNone(self.process.date_created)
self.assertEqual(self.process.state, Process.QUEUED)
self.assertEqual(self.process.details['course_list'], [123, 124, 125])
self._flush_jobs()
self._refresh_from_db()
self.assertEqual(self.process.state, Process.COMPLETE)
self.assertEqual(self.process.details['course_list'], [123, 124, 125])
self.assertEqual(self.process.details['op_config']['courses'],
[123, 124, 125])
def run(request):
if request.method == 'POST':
print 'got run request for makecatalogs'
# remove records for exited processes
Process.objects.filter(name='makecatalogs', exited=True).delete()
while True:
# Loop until there are no more running processes
processes = Process.objects.filter(name='makecatalogs',
exited=False)
if not processes:
break
# clean up any processes no longer in the process table
for process in processes:
if not pid_exists(process.pid):
process.delete()
processes = Process.objects.filter(name='makecatalogs',
exited=False)
if not processes:
break
time.sleep(1)
proc = subprocess.Popen(
[MAKECATALOGS, REPO_DIR],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
record = Process(name='makecatalogs')
record.pid = proc.pid
record.save()
while True:
output = proc.stdout.readline().decode('utf-8').rstrip('\n')
if output:
record.statustext = output.rstrip('\n')
record.save()
if proc.poll() != None:
break
record.statustext = 'Done'
record.exited = True
record.exitcode = proc.returncode
record.save()
return HttpResponse(json.dumps('done'),
content_type='application/json')
return HttpResponse(json.dumps('must be a POST request'),
content_type='application/json')
def load_all_fixtures(db, app):
user_datastore = SQLAlchemyUserDatastore(db, User, Role)
# Load default fixtures
role = user_datastore.create_role(name=app.config['VOXO_ROLE_USER'],
description='User role')
admin_role = user_datastore.create_role(name=app.config['VOXO_ROLE_ADMIN'],
description='Admin role')
server_role = user_datastore.create_role(
name=app.config['VOXO_ROLE_SERVER'], description='Server role')
user_vjousse = registerable.register_user(email='*****@*****.**',
password='******')
user_datastore.add_role_to_user(user_vjousse, role)
print("{user} with token {token} created.".format(
user=user_vjousse.email, token=user_vjousse.get_auth_token()))
user = registerable.register_user(email='*****@*****.**',
password='******')
user_datastore.add_role_to_user(user, admin_role)
user_datastore.add_role_to_user(user, role)
print("{user} with token {token} created.".format(
user=user.email, token=user.get_auth_token()))
server = registerable.register_user(email='*****@*****.**',
password='******')
user_datastore.add_role_to_user(server, server_role)
print("{user} with token {token} created.".format(
user=server.email, token=server.get_auth_token()))
asr_model_french = AsrModel(name="french.studio.fr_FR",
description="General purpose french model")
asr_model_french.users.append(user_vjousse)
db.session.add(asr_model_french)
db.session.flush()
asr_model_english = AsrModel(name="english.studio",
description="General purpose english model")
asr_model_english.users.append(user_vjousse)
db.session.add(asr_model_english)
db.session.flush()
media_file = MediaFile(filename='fixture_file.wav',
status=FileStatus.Success,
user_id=user_vjousse.id,
size=2500,
generated_filename='fixture_file_UUID.wav',
duration=70)
db.session.add(media_file)
db.session.flush()
process = Process(
file_id=media_file.id,
status=DecodeStatus.Queued,
)
db.session.add(process)
db.session.flush()
transcription = Transcription(
auto_filename='transcription.xml',
ref_filename='transcription.txt',
user_id=user_vjousse.id,
)
db.session.add(transcription)
db.session.flush()
process = Process(file_id=media_file.id,
status=DecodeStatus.Queued,
type=ProcessType.TranscriptionAlignment,
transcription_id=transcription.id)
db.session.add(process)
db.session.flush()
db.session.commit()
def queued_processes_list(db: Session = Depends(deps.get_db)):
"""
List queued processes
"""
return jsonable_encoder(Process.queued_processes_list(db=db))
def process_output_list(id: int, db: Session = Depends(deps.get_db)):
"""
Get process output detail
"""
return jsonable_encoder(Process.process_status_detail(db=db, id=id))
def run(request):
if request.method == 'POST':
print 'got run request for makecatalogs'
# remove records for exited processes
Process.objects.filter(name='makecatalogs', exited=True).delete()
while True:
# Loop until there are no more running processes
processes = Process.objects.filter(name='makecatalogs',
exited=False)
if not processes:
break
# clean up any processes no longer in the process table
for process in processes:
if not pid_exists(process.pid):
process.delete()
processes = Process.objects.filter(name='makecatalogs',
exited=False)
if not processes:
break
time.sleep(1)
proc = subprocess.Popen([MAKECATALOGS, REPO_DIR],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
record = Process(name='makecatalogs')
record.pid = proc.pid
record.save()
while True:
output = proc.stdout.readline().decode('utf-8').rstrip('\n')
if output:
record.statustext = output.rstrip('\n')
record.save()
if proc.poll() != None:
break
record.statustext = 'Done'
record.exited = True
record.exitcode = proc.returncode
record.save()
return HttpResponse(json.dumps('done'),
content_type='application/json')
return HttpResponse(json.dumps('must be a POST request'),
content_type='application/json')
def processes(measurement):
for proc in psutil.process_iter(attrs=['cpu_percent']):
pass
sleep(1)
attrs = [
'pid',
'name',
'exe',
'cmdline',
'username',
'nice',
'ionice',
'io_counters',
'num_ctx_switches',
'num_fds',
'num_threads',
'cpu_percent',
'memory_full_info',
'status',
]
for item in psutil.process_iter(attrs=attrs):
process = Process(
measurement=measurement,
pid=item.info['pid'],
name=item.info['name'],
exe=item.info['exe'],
cmdline=' '.join(item.info['cmdline']),
username=item.info['username'],
nice=item.info['nice'],
num_fds=item.info['num_fds'],
num_threads=item.info['num_threads'],
cpu_percent=item.info['cpu_percent'],
status=item.info['status'],
)
if item.info['memory_full_info'] is not None:
ProcessMemory(
process=process,
rss=item.info['memory_full_info'].rss,
vms=item.info['memory_full_info'].vms,
shared=item.info['memory_full_info'].shared,
text=item.info['memory_full_info'].text,
lib=item.info['memory_full_info'].lib,
data=item.info['memory_full_info'].data,
dirty=item.info['memory_full_info'].dirty,
uss=item.info['memory_full_info'].uss,
pss=item.info['memory_full_info'].pss,
swap=item.info['memory_full_info'].swap,
)
ProcessIONice(
process=process,
ioclass=int(item.info['ionice'].ioclass),
value=item.info['ionice'].value,
)
if item.info['io_counters'] is not None:
ProcessIOCounters(
process=process,
read_count=item.info['io_counters'].read_count,
write_count=item.info['io_counters'].write_count,
read_bytes=item.info['io_counters'].read_bytes,
write_bytes=item.info['io_counters'].write_bytes,
read_chars=item.info['io_counters'].read_chars,
write_chars=item.info['io_counters'].write_chars,
)
ProcessContextSwitches(
process=process,
voluntary=item.info['num_ctx_switches'].voluntary,
involuntary=item.info['num_ctx_switches'].involuntary,
)
